From 6cd4ba5dc4b60dadc65325b217346f157113a3e0 Mon Sep 17 00:00:00 2001 From: Suhail Hany Date: Sat, 19 Sep 2026 14:55:21 +0300 Subject: [PATCH 001/229] difftool: do not crash on a path rejected by verify_path() `git difftool --dir-diff` checks both sides of the diff out into temporary directories. For every path in the raw diff, checkout_path() builds a transient cache entry and hands it to checkout_entry(): ce = make_transient_cache_entry(mode, oid, path, 0, NULL); ret = checkout_entry(ce, state, NULL, NULL); make_transient_cache_entry() runs the path through verify_path() and, when that fails, reports "invalid path '%s'" and returns NULL. checkout_path() never checks for that, so checkout_entry_ca() dereferences a NULL cache entry and Git dies with a segmentation fault (access violation 0xC0000005 on Windows) immediately after printing the error. Any diff whose raw output contains a path verify_path() rejects is enough to reach it. A tree with a top-level ".git" entry is the shortest reproducer: blob=$(echo content | git hash-object -w --stdin) tree=$(printf "100644 blob %s\t.git\n" "$blob" | git mktree) commit=$(git commit-tree -m bad "$tree") git difftool --dir-diff --no-symlinks --extcmd true "$commit" HEAD Check for NULL and return an error instead. builtin/checkout.c, the only other caller of make_transient_cache_entry(), die()s here, but checkout_path() already returns int and both of its call sites turn a non-zero return into "could not write ''" before jumping to the cleanup label, which exits with status 1. Reusing that path keeps the diagnosis that make_transient_cache_entry() already printed and turns the crash into an ordinary failure. Signed-off-by: Suhail Hany --- builtin/difftool.c | 2 ++ t/t7800-difftool.sh | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/builtin/difftool.c b/builtin/difftool.c index bc7b2ea44396bb..8208ade4727074 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -336,6 +336,8 @@ static int checkout_path(unsigned mode, struct object_id *oid, int ret; ce = make_transient_cache_entry(mode, oid, path, 0, NULL); + if (!ce) + return -1; ret = checkout_entry(ce, state, NULL, NULL); discard_cache_entry(ce); diff --git a/t/t7800-difftool.sh b/t/t7800-difftool.sh index 0079aa9da440cf..40983602b7e9a6 100755 --- a/t/t7800-difftool.sh +++ b/t/t7800-difftool.sh @@ -668,6 +668,15 @@ run_dir_diff_test 'difftool --dir-diff with no diff' ' git difftool -d main main ' +test_expect_success 'difftool --dir-diff errors out on an invalid path' ' + blob=$(echo content | git hash-object -w --stdin) && + tree=$(printf "100644 blob %s\t.git\n" "$blob" | git mktree) && + commit=$(git commit-tree -m "tree with a .git entry" "$tree") && + test_must_fail git difftool --dir-diff --no-symlinks \ + --extcmd true "$commit" HEAD 2>err && + test_grep "invalid path" err +' + write_script modify-file <<\EOF echo "new content" >file EOF From 183b4d62d62ba32354ec4b7cf73703fba3c095d7 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sun, 24 Jul 2011 15:54:04 +0200 Subject: [PATCH 002/229] t9350: point out that refs are not updated correctly This happens only when the corresponding commits are not exported in the current fast-export run. This can happen either when the relevant commit is already marked, or when the commit is explicitly marked as UNINTERESTING with a negative ref by another argument. This breaks fast-export basec remote helpers. Signed-off-by: Sverre Rabbelier --- t/t9350-fast-export.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t9350-fast-export.sh b/t/t9350-fast-export.sh index a487476ad45beb..f4428b21e3211a 100755 --- a/t/t9350-fast-export.sh +++ b/t/t9350-fast-export.sh @@ -1010,4 +1010,15 @@ test_expect_success GPG,RUST 'export and import of doubly signed commit' ' fi ' +test_expect_failure 'refs are updated even if no commits need to be exported' ' + cat > expected <<-EOF && + reset refs/heads/main + from $(git rev-parse main) + + EOF + + git fast-export main..main > actual && + test_cmp expected actual +' + test_done From 40d550a61985727c47d0b42fe668b9a41a793ffa Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 28 Sep 2026 14:43:54 +0200 Subject: [PATCH 003/229] difftool: avoid memory leak when `run_dir_diff()` fails Some error code paths return before the child process was spawned; In these instances, the `child_process` struct is still initialized and still holds allocated data, which would cause the sanitizer (which is exercised in the `*-leaks` CI jobs) to complain loudly. The parent commit introduced a test case to verify that invalid paths won't let `difftool` crash with a segmentation fault; That test case (inadvertently) exercises the very issue this here patch addresses in the `-leaks` CI jobs. Signed-off-by: Johannes Schindelin --- builtin/difftool.c | 1 + 1 file changed, 1 insertion(+) diff --git a/builtin/difftool.c b/builtin/difftool.c index 8208ade4727074..07e9f4409c2a6f 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -696,6 +696,7 @@ static int run_dir_diff(struct repository *repo, strbuf_release(&wtdir); strbuf_release(&buf); strbuf_release(&tmpdir); + child_process_clear(child); return (ret < 0) ? 1 : ret; } From aadab5db56be1f29de336e9bc180092c76799e9a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 22 Jun 2026 20:32:58 +0000 Subject: [PATCH 004/229] ci: bump actions/checkout from 6 to 7 The primary change in this version jump (and the reason for a _major_ version increment) is that this Action is now disabled in GitHub workflows triggered by workflow_run and pull_request_target events, none of which are used in Git's workflows. For more details, see https://github.com/actions/checkout/releases. Originally-authored-by: dependabot[bot] Signed-off-by: Johannes Schindelin --- .github/workflows/check-style.yml | 2 +- .github/workflows/check-whitespace.yml | 2 +- .github/workflows/coverity.yml | 2 +- .github/workflows/main.yml | 24 ++++++++++++------------ 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/check-style.yml b/.github/workflows/check-style.yml index 108a2de903310c..72e66762601413 100644 --- a/.github/workflows/check-style.yml +++ b/.github/workflows/check-style.yml @@ -20,7 +20,7 @@ jobs: jobname: ClangFormat runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/check-whitespace.yml b/.github/workflows/check-whitespace.yml index ea6f49f742108e..7a35b67a636acc 100644 --- a/.github/workflows/check-whitespace.yml +++ b/.github/workflows/check-whitespace.yml @@ -19,7 +19,7 @@ jobs: check-whitespace: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 with: fetch-depth: 0 diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 58a78f1eb3f836..e0ee69acb3442f 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -38,7 +38,7 @@ jobs: COVERITY_LANGUAGE: cxx COVERITY_PLATFORM: overridden-below steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: install minimal Git for Windows SDK if: contains(matrix.os, 'windows') uses: git-for-windows/setup-git-for-windows-sdk@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b229739be8b5ec..0436cdb6fe40e6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -114,7 +114,7 @@ jobs: group: windows-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: git-for-windows/setup-git-for-windows-sdk@v2 - name: Install GCC-compatible Rust target shell: bash @@ -199,10 +199,10 @@ jobs: group: vs-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: git-for-windows/setup-git-for-windows-sdk@v2 - name: initialize vcpkg - uses: actions/checkout@v6 + uses: actions/checkout@v7 with: repository: 'microsoft/vcpkg' path: 'compat/vcbuild/vcpkg' @@ -284,7 +284,7 @@ jobs: group: windows-meson-build-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 - name: Set up dependencies shell: pwsh @@ -312,7 +312,7 @@ jobs: group: windows-meson-test-${{ matrix.nr }}-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - uses: actions/setup-python@v6 - name: Set up dependencies shell: pwsh @@ -367,7 +367,7 @@ jobs: TEST_OUTPUT_DIRECTORY: ${{github.workspace}}/t runs-on: ${{matrix.vector.pool}} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - name: print test failures @@ -388,7 +388,7 @@ jobs: CI_JOB_IMAGE: ubuntu-latest runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - run: ci/install-dependencies.sh - run: ci/run-build-and-minimal-fuzzers.sh dockerized: @@ -467,7 +467,7 @@ jobs: else apt-get -q update && apt-get -q -y install git fi - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - run: ci/install-dependencies.sh - run: useradd builder --create-home - run: chown -R builder . @@ -492,7 +492,7 @@ jobs: group: static-analysis-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - run: ci/install-dependencies.sh - run: ci/run-static-analysis.sh - run: ci/check-directional-formatting.bash @@ -508,7 +508,7 @@ jobs: group: rust-analysis-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - run: ci/install-dependencies.sh - run: ci/run-rust-checks.sh sparse: @@ -522,7 +522,7 @@ jobs: group: sparse-${{ github.ref }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Install other dependencies run: ci/install-dependencies.sh - run: make sparse @@ -538,6 +538,6 @@ jobs: CI_JOB_IMAGE: ubuntu-latest runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - run: ci/install-dependencies.sh - run: ci/test-documentation.sh From ee4ff4582aeb256cbdd8c0aa679d3e677ddfce63 Mon Sep 17 00:00:00 2001 From: Jeff King Date: Mon, 13 Jan 2025 01:26:01 -0500 Subject: [PATCH 005/229] grep: prevent `^$` false match at end of file In some implementations, `regexec_buf()` assumes that it is fed lines; Without `REG_NOTEOL` it thinks the end of the buffer is the end of a line. Which makes sense, but trips up this case because we are not feeding lines, but rather a whole buffer. So the final newline is not the start of an empty line, but the true end of the buffer. This causes an interesting bug: $ echo content >file.txt $ git grep --no-index -n '^$' file.txt file.txt:2: This bug is fixed by making the end of the buffer consistently the end of the final line. The patch was applied from https://lore.kernel.org/git/20250113062601.GD767856@coredump.intra.peff.net/ Reported-by: Olly Betts Signed-off-by: Johannes Schindelin --- grep.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/grep.c b/grep.c index 1d75d314211aa8..733fd3a80033e2 100644 --- a/grep.c +++ b/grep.c @@ -1647,6 +1647,8 @@ static int grep_source_1(struct grep_opt *opt, struct grep_source *gs, int colle bol = gs->buf; left = gs->size; + if (left && gs->buf[left-1] == '\n') + left--; while (left) { const char *eol; int hit; From 838e76003865294374666d7ca87fdd77b0160781 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 30 Oct 2024 19:48:46 +0100 Subject: [PATCH 006/229] unix-socket: avoid leak when initialization fails When a Unix socket is initialized, the current directory's path is stored so that the cleanup code can `chdir()` back to where it was before exit. If the path that needs to be stored exceeds the default size of the `sun_path` attribute of `struct sockaddr_un` (which is defined as a 108-sized byte array on Linux), a larger buffer needs to be allocated so that it can hold the path, and it is the responsibility of the `unix_sockaddr_cleanup()` function to release that allocated memory. In Git's CI, this stack allocation is not necessary because the code is checked out to `/home/runner/work/git/git`. Concatenate the path `t/trash directory.t0301-credential-cache/.cache/git/credential/socket` and a terminating NUL, and you end up with 96 bytes, 12 shy of the default `sun_path` size. However, I use worktrees with slightly longer paths: `/home/me/projects/git/yes/i/nest/worktrees/to/organize/them/` is more in line with what I have. When I recently tried to locally reproduce a failure of the `linux-leaks` CI job, this t0301 test failed (where it had not failed in CI). The reason: When `credential-cache` tries to reach its daemon initially by calling `unix_sockaddr_init()`, it is expected that the daemon cannot be reached (the idea is to spin up the daemon in that case and try again). However, when this first call to `unix_sockaddr_init()` fails, the code returns early from the `unix_stream_connect()` function _without_ giving the cleanup code a chance to run, skipping the deallocation of above-mentioned path. The fix is easy: do not return early but instead go directly to the cleanup code. Signed-off-by: Johannes Schindelin --- unix-socket.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unix-socket.c b/unix-socket.c index 8860203c3f46dc..1fa0cf6c15c721 100644 --- a/unix-socket.c +++ b/unix-socket.c @@ -84,7 +84,7 @@ int unix_stream_connect(const char *path, int disallow_chdir) struct unix_sockaddr_context ctx; if (unix_sockaddr_init(&sa, path, &ctx, disallow_chdir) < 0) - return -1; + goto fail; fd = socket(AF_UNIX, SOCK_STREAM, 0); if (fd < 0) goto fail; From e61cde7908dab100c3ea0ac47eff78a15af010e0 Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sat, 28 Aug 2010 20:49:01 -0500 Subject: [PATCH 007/229] transport-helper: add trailing -- [PT: ensure we add an additional element to the argv array] Signed-off-by: Sverre Rabbelier Signed-off-by: Johannes Schindelin --- transport-helper.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transport-helper.c b/transport-helper.c index 06765febd6fa6d..4dda7df7de9aef 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -501,6 +501,8 @@ static int get_exporter(struct transport *transport, for (size_t i = 0; i < revlist_args->nr; i++) strvec_push(&fastexport->args, revlist_args->items[i].string); + strvec_push(&fastexport->args, "--"); + fastexport->git_cmd = 1; return start_command(fastexport); } From 83f4f16f998bc2b3ae96fdcff9c327ccb4151a69 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 28 Sep 2026 20:51:17 +0200 Subject: [PATCH 008/229] difftool: avoid invalid memory access by an atexit handler The atexit handler `cleanup_children()` would happily dereference the stored `process` of the `child` command, unless `finish_command()` took that process off the list that this atexit handler processes. Therefore we need to ensure that `finish_command()` is called. In case that an invalid path is encountered, this is currently not the case: the process was started, but `checkout_path()` encountered an error and `goto finish` ensures an early exit. As running t7800 with `--valgrind-only=83` demonstrates, there is then an invalid memory access in the atexit handler that tries to dereference the `child_process` instance that was on the stack and is now long gone. Let's use `fp` (the `FILE *` corresponding to the output pipe) as a tell-tale that the process was started, but `finish_command()` was not called yet: `fp` is set to non-`NULL` only after a successful `start_command()`, and it is re-set to `NULL` after a `finish_command()`. Therefore, also call `finish_command()` in addition to closing the not-yet-closed output pipe. Signed-off-by: Johannes Schindelin --- builtin/difftool.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/builtin/difftool.c b/builtin/difftool.c index 07e9f4409c2a6f..991d298584d1be 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -677,8 +677,17 @@ static int run_dir_diff(struct repository *repo, } finish: - if (fp) + if (fp) { fclose(fp); + /* + * The child process was started, but there was an error before + * it finished; After closing the output pipe, we need to wait + * for the process to finish, otherwise the atexit handler + * `cleanup_children()` would try to access the stack-allocated + * `child`. + */ + finish_command(child); /* best effort; ignore result */ + } hashmap_clear_and_free(&working_tree_dups, struct working_tree_entry, entry); hashmap_clear_and_free(&wt_modified, struct path_entry, entry); From 3d90764ec43b59154e8abe6611501be2b2fd92e6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 26 Jun 2026 12:15:27 +0200 Subject: [PATCH 009/229] bundle-uri: refuse advertised URIs by protocol Servers may advertise bundle URIs that are not HTTP(S); copy_uri_to_file() then opens it as a local path. On Windows that can be a UNC path like `//attacker/share/x`, i.e. a clone can be manipulated into making an outbound SMB connection that leaks NTLM credentials (CVE-2026-62960). Subject advertised URIs to the usual protocol allow-list (`protocol.*.allow`), which drops "file" (and bare/UNC paths) by default but keeps http/https/git/ssh. Do it in fetch_bundle_list(), the clone/fetch consume path, so ls-remote still lists everything; each skipped URI is reported. A user-supplied `--bundle-uri` is unaffected, and `protocol.file.allow=always` re-enables an advertised file URI. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- bundle-uri.c | 50 ++++++++++++++++++++++++++++++++ t/lib-bundle-uri-protocol.sh | 56 ++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/bundle-uri.c b/bundle-uri.c index 2bb2eb99e47b58..92a36ca0ab3a43 100644 --- a/bundle-uri.c +++ b/bundle-uri.c @@ -15,6 +15,8 @@ #include "remote.h" #include "trace2.h" #include "odb.h" +#include "transport.h" +#include "url.h" static struct { enum bundle_list_heuristic heuristic; @@ -890,11 +892,59 @@ int fetch_bundle_uri(struct repository *r, const char *uri, return result; } +/* protocol of 'uri', or "file" if it has none (bare/UNC/relative path) */ +static void bundle_uri_protocol(const char *uri, struct strbuf *out) +{ + const char *p = uri; + + while (is_urlschemechar(p == uri, *p)) + p++; + strbuf_reset(out); + if (p > uri && starts_with(p, "://")) + strbuf_add(out, uri, p - uri); + else + strbuf_addstr(out, "file"); +} + +/* Drop advertised URIs whose protocol is not allowed (see protocol.*.allow). */ +static void sanitize_bundle_list(struct bundle_list *list) +{ + struct remote_bundle_info **skipped; + size_t nr = 0, i; + struct remote_bundle_info *info; + struct hashmap_iter iter; + struct strbuf proto = STRBUF_INIT; + + ALLOC_ARRAY(skipped, hashmap_get_size(&list->bundles)); + hashmap_for_each_entry(&list->bundles, &iter, info, ent) { + if (!info->uri) + continue; + bundle_uri_protocol(info->uri, &proto); + /* advertised URIs are not user-provided */ + if (!is_transport_allowed(proto.buf, 0)) { + warning(_("skipping bundle URI '%s': protocol '%s' " + "is not allowed"), info->uri, proto.buf); + skipped[nr++] = info; + } + } + strbuf_release(&proto); + + for (i = 0; i < nr; i++) { + hashmap_remove(&list->bundles, &skipped[i]->ent, NULL); + clear_remote_bundle_info(skipped[i], NULL); + free(skipped[i]); + } + + free(skipped); +} + int fetch_bundle_list(struct repository *r, struct bundle_list *list) { int result; struct bundle_list global_list; + sanitize_bundle_list(list); + /* * If the creationToken heuristic is used, then the URIs * advertised by 'list' are not nested lists and instead diff --git a/t/lib-bundle-uri-protocol.sh b/t/lib-bundle-uri-protocol.sh index 794478ae194c8d..889e673a442f77 100644 --- a/t/lib-bundle-uri-protocol.sh +++ b/t/lib-bundle-uri-protocol.sh @@ -237,3 +237,59 @@ test_expect_success "test bundle-uri with $BUNDLE_URI_PROTOCOL:// using protocol >actual && test_cmp_config_output expect actual ' + +# Advertised bundle URIs are subject to protocol.*.allow; "file" (and bare or +# UNC paths) is denied by default, so such a URI must be skipped, not fetched. +advertise_uri () { + test_config -C "$BUNDLE_URI_PARENT" bundle.version 1 && + test_config -C "$BUNDLE_URI_PARENT" bundle.mode all && + test_config -C "$BUNDLE_URI_PARENT" bundle.payload.uri "$1" +} + +ignores_advertised_uri () { + rm -rf victim && + advertise_uri "$1" && + git -c transfer.bundleURI=true -c protocol.version=2 \ + clone "$BUNDLE_URI_REPO_URI" victim && + git -C victim for-each-ref refs/bundles/ >refs && + test_must_be_empty refs +} + +test_expect_success "create bundle to advertise" ' + git -C "$BUNDLE_URI_PARENT" bundle create "$PWD/payload.bundle" main +' + +test_expect_success "ignore non-HTTP(S) bundle URI with $BUNDLE_URI_PROTOCOL://" ' + ignores_advertised_uri "$PWD/payload.bundle" && + ignores_advertised_uri "file://$PWD/payload.bundle" +' + +test_expect_success "protocol.file.allow=always honors file bundle URI with $BUNDLE_URI_PROTOCOL://" ' + rm -rf victim && + advertise_uri "$PWD/payload.bundle" && + git -c transfer.bundleURI=true -c protocol.version=2 \ + -c protocol.file.allow=always \ + clone "$BUNDLE_URI_REPO_URI" victim && + git -C victim rev-parse --verify refs/bundles/heads/main +' + +# same path via a UNC administrative share (cf. t5580-unc-paths.sh) +if test_have_prereq CYGWIN +then + UNCPATH="$(cygpath -aw .)" +elif test_have_prereq MINGW +then + UNCPATH="$(pwd)" +fi +case "$UNCPATH" in +[A-Za-z]:*) + WITHOUTDRIVE="${UNCPATH#?:}" + UNCPATH="//localhost/${UNCPATH%%:*}\$$WITHOUTDRIVE" + test -d "$UNCPATH" && test_set_prereq ADMIN_UNC + ;; +esac + +test_expect_success ADMIN_UNC "ignore UNC bundle URI with $BUNDLE_URI_PROTOCOL://" ' + ignores_advertised_uri "$UNCPATH/payload.bundle" && + ignores_advertised_uri "file://$UNCPATH/payload.bundle" +' From f01fa763ff3b446d6f1615129dea8c3681a1f1c9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 29 Jun 2026 20:32:59 +0000 Subject: [PATCH 010/229] ci(coverity): bump actions/cache from 5 to 6 This update only means that the Action is now an ECMAScript module, no functional changes implied. Originally-authored-by: dependabot[bot] Signed-off-by: Johannes Schindelin --- .github/workflows/coverity.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index e0ee69acb3442f..92438f5c7d934c 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -98,7 +98,7 @@ jobs: # A cache miss will add ~30s to create, but a cache hit will save minutes. - name: restore the Coverity Build Tool id: cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@v6 with: path: ${{ runner.temp }}/cov-analysis key: cov-build-${{ env.COVERITY_LANGUAGE }}-${{ env.COVERITY_PLATFORM }}-${{ steps.lookup.outputs.hash }} @@ -141,7 +141,7 @@ jobs: esac - name: cache the Coverity Build Tool if: steps.cache.outputs.cache-hit != 'true' - uses: actions/cache/save@v5 + uses: actions/cache/save@v6 with: path: ${{ runner.temp }}/cov-analysis key: cov-build-${{ env.COVERITY_LANGUAGE }}-${{ env.COVERITY_PLATFORM }}-${{ steps.lookup.outputs.hash }} From d2daf46c7fdc396c2605bf281cc0951d27d8f3c4 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 30 Mar 2021 14:25:31 -0400 Subject: [PATCH 011/229] clink.pl: fix libexpatd.lib link error when using MSVC When building with `make MSVC=1 DEBUG=1`, link to `libexpatd.lib` rather than `libexpat.lib`. It appears that the `vcpkg` package for "libexpat" has changed and now creates `libexpatd.lib` for debug mode builds. Previously, both debug and release builds created a ".lib" with the same basename. Signed-off-by: Jeff Hostetler --- compat/vcbuild/scripts/clink.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 3bd824154be381..2768ae15f1879f 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -66,7 +66,11 @@ } push(@args, $lib); } elsif ("$arg" eq "-lexpat") { + if ($is_debug) { + push(@args, "libexpatd.lib"); + } else { push(@args, "libexpat.lib"); + } } elsif ("$arg" =~ /^-L/ && "$arg" ne "-LTCG") { $arg =~ s/^-L/-LIBPATH:/; push(@lflags, $arg); From 223115de8159bc6725bfa797ab9f21255e11fde3 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 5 Apr 2021 15:27:38 -0400 Subject: [PATCH 012/229] Makefile: clean up .ilk files when MSVC=1 Signed-off-by: Jeff Hostetler --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 984559fd680f09..1d44599dc6659a 100644 --- a/Makefile +++ b/Makefile @@ -3973,12 +3973,15 @@ ifdef MSVC $(RM) $(patsubst %.o,%.o.pdb,$(OBJECTS)) $(RM) headless-git.o.pdb $(RM) $(patsubst %.exe,%.pdb,$(OTHER_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ilk,$(OTHER_PROGRAMS)) $(RM) $(patsubst %.exe,%.iobj,$(OTHER_PROGRAMS)) $(RM) $(patsubst %.exe,%.ipdb,$(OTHER_PROGRAMS)) $(RM) $(patsubst %.exe,%.pdb,$(PROGRAMS)) + $(RM) $(patsubst %.exe,%.ilk,$(PROGRAMS)) $(RM) $(patsubst %.exe,%.iobj,$(PROGRAMS)) $(RM) $(patsubst %.exe,%.ipdb,$(PROGRAMS)) $(RM) $(patsubst %.exe,%.pdb,$(TEST_PROGRAMS)) + $(RM) $(patsubst %.exe,%.ilk,$(TEST_PROGRAMS)) $(RM) $(patsubst %.exe,%.iobj,$(TEST_PROGRAMS)) $(RM) $(patsubst %.exe,%.ipdb,$(TEST_PROGRAMS)) $(RM) compat/vcbuild/MSVC-DEFS-GEN From 2e3e628c710e99a93475948418c5bb10e4753e35 Mon Sep 17 00:00:00 2001 From: Ian Bearman Date: Fri, 31 Jan 2020 16:00:25 -0800 Subject: [PATCH 013/229] vcbuild: install ARM64 dependencies when building ARM64 binaries Co-authored-by: Dennis Ameling Signed-off-by: Ian Bearman Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- compat/vcbuild/README | 6 +++++- compat/vcbuild/vcpkg_copy_dlls.bat | 7 ++++++- compat/vcbuild/vcpkg_install.bat | 9 +++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/compat/vcbuild/README b/compat/vcbuild/README index 29ec1d0f104b80..1df1cabb1ebbbd 100644 --- a/compat/vcbuild/README +++ b/compat/vcbuild/README @@ -6,7 +6,11 @@ The Steps to Build Git with VS2015 or VS2017 from the command line. Prompt or from an SDK bash window: $ cd - $ ./compat/vcbuild/vcpkg_install.bat + $ ./compat/vcbuild/vcpkg_install.bat x64-windows + + or + + $ ./compat/vcbuild/vcpkg_install.bat arm64-windows The vcpkg tools and all of the third-party sources will be installed in this folder: diff --git a/compat/vcbuild/vcpkg_copy_dlls.bat b/compat/vcbuild/vcpkg_copy_dlls.bat index 13661c14f8705c..8bea0cbf83b6cf 100644 --- a/compat/vcbuild/vcpkg_copy_dlls.bat +++ b/compat/vcbuild/vcpkg_copy_dlls.bat @@ -15,7 +15,12 @@ REM ================================================================ @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD cd %cwd% - SET arch=x64-windows + SET arch=%2 + IF NOT DEFINED arch ( + echo defaulting to 'x64-windows`. Invoke %0 with 'x86-windows', 'x64-windows', or 'arm64-windows' + set arch=x64-windows + ) + SET inst=%cwd%vcpkg\installed\%arch% IF [%1]==[release] ( diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index 8330d8120fb511..cacef18c11dc79 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -31,6 +31,12 @@ REM ================================================================ SETLOCAL EnableDelayedExpansion + SET arch=%1 + IF NOT DEFINED arch ( + echo defaulting to 'x64-windows`. Invoke %0 with 'x86-windows', 'x64-windows', or 'arm64-windows' + set arch=x64-windows + ) + @FOR /F "delims=" %%D IN ("%~dp0") DO @SET cwd=%%~fD cd %cwd% @@ -55,9 +61,8 @@ REM ================================================================ echo Successfully installed %cwd%vcpkg\vcpkg.exe :install_libraries - SET arch=x64-windows - echo Installing third-party libraries... + echo Installing third-party libraries(%arch%)... FOR %%i IN (zlib expat libiconv openssl libssh2 curl) DO ( cd %cwd%vcpkg IF NOT EXIST "packages\%%i_%arch%" CALL :sub__install_one %%i From b8929cc36852d7eca10dad69e04a042aa69a5c1c Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 5 Apr 2021 14:08:22 -0400 Subject: [PATCH 014/229] vcbuild: add support for compiling Windows resource files Create a wrapper for the Windows Resource Compiler (RC.EXE) for use by the MSVC=1 builds. This is similar to the CL.EXE and LIB.EXE wrappers used for the MSVC=1 builds. Signed-off-by: Jeff Hostetler --- compat/vcbuild/find_vs_env.bat | 7 ++++++ compat/vcbuild/scripts/rc.pl | 46 ++++++++++++++++++++++++++++++++++ config.mak.uname | 3 ++- 3 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 compat/vcbuild/scripts/rc.pl diff --git a/compat/vcbuild/find_vs_env.bat b/compat/vcbuild/find_vs_env.bat index b35d264c0e6bed..379b16296e09c2 100644 --- a/compat/vcbuild/find_vs_env.bat +++ b/compat/vcbuild/find_vs_env.bat @@ -99,6 +99,7 @@ REM ================================================================ SET sdk_dir=%WindowsSdkDir% SET sdk_ver=%WindowsSDKVersion% + SET sdk_ver_bin_dir=%WindowsSdkVerBinPath%%tgt% SET si=%sdk_dir%Include\%sdk_ver% SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" SET sl=%sdk_dir%lib\%sdk_ver% @@ -130,6 +131,7 @@ REM ================================================================ SET sdk_dir=%WindowsSdkDir% SET sdk_ver=%WindowsSDKVersion% + SET sdk_ver_bin_dir=%WindowsSdkVerBinPath%bin\amd64 SET si=%sdk_dir%Include\%sdk_ver% SET sdk_includes=-I"%si%ucrt" -I"%si%um" -I"%si%shared" -I"%si%winrt" SET sl=%sdk_dir%lib\%sdk_ver% @@ -160,6 +162,11 @@ REM ================================================================ echo msvc_includes=%msvc_includes% echo msvc_libs=%msvc_libs% + echo sdk_ver_bin_dir=%sdk_ver_bin_dir% + SET X1=%sdk_ver_bin_dir:C:=/C% + SET X2=%X1:\=/% + echo sdk_ver_bin_dir_msys=%X2% + echo sdk_includes=%sdk_includes% echo sdk_libs=%sdk_libs% diff --git a/compat/vcbuild/scripts/rc.pl b/compat/vcbuild/scripts/rc.pl new file mode 100644 index 00000000000000..7bca4cd81c6c63 --- /dev/null +++ b/compat/vcbuild/scripts/rc.pl @@ -0,0 +1,46 @@ +#!/usr/bin/perl -w +###################################################################### +# Compile Resources on Windows +# +# This is a wrapper to facilitate the compilation of Git with MSVC +# using GNU Make as the build system. So, instead of manipulating the +# Makefile into something nasty, just to support non-space arguments +# etc, we use this wrapper to fix the command line options +# +###################################################################### +use strict; +my @args = (); +my @input = (); + +while (@ARGV) { + my $arg = shift @ARGV; + if ("$arg" =~ /^-[dD]/) { + # GIT_VERSION gets passed with too many + # layers of dquote escaping. + $arg =~ s/\\"/"/g; + + push(@args, $arg); + + } elsif ("$arg" eq "-i") { + my $arg = shift @ARGV; + # TODO complain if NULL or is dashed ?? + push(@input, $arg); + + } elsif ("$arg" eq "-o") { + my $arg = shift @ARGV; + # TODO complain if NULL or is dashed ?? + push(@args, "-fo$arg"); + + } else { + push(@args, $arg); + } +} + +push(@args, "-nologo"); +push(@args, "-v"); +push(@args, @input); + +unshift(@args, "rc.exe"); +printf("**** @args\n"); + +exit (system(@args) != 0); diff --git a/config.mak.uname b/config.mak.uname index 820c5c0e49084a..152ff78fe6d731 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -477,7 +477,7 @@ ifeq ($(uname_S),Windows) # link.exe next to, and required by, cl.exe, we have to prepend this # onto the existing $PATH. # - SANE_TOOL_PATH ?= $(msvc_bin_dir_msys) + SANE_TOOL_PATH ?= $(msvc_bin_dir_msys):$(sdk_ver_bin_dir_msys) HAVE_ALLOCA_H = YesPlease NO_PREAD = YesPlease NO_WRITEV = YesPlease @@ -548,6 +548,7 @@ endif # See https://msdn.microsoft.com/en-us/library/ms235330.aspx EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib PTHREAD_LIBS = + RC = compat/vcbuild/scripts/rc.pl lib = BASIC_CFLAGS += $(vcpkg_inc) $(sdk_includes) $(msvc_includes) ifndef DEBUG From d87e98136aea30e4ad6da9a222eeefd18159620d Mon Sep 17 00:00:00 2001 From: Ian Bearman Date: Tue, 4 Feb 2020 10:34:40 -0800 Subject: [PATCH 015/229] vcbuild: add an option to install individual 'features' In this context, a "feature" is a dependency combined with its own dependencies. Signed-off-by: Ian Bearman Signed-off-by: Johannes Schindelin --- compat/vcbuild/vcpkg_install.bat | 35 +++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index cacef18c11dc79..8da212487ae97d 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -85,14 +85,47 @@ REM ================================================================ :sub__install_one echo Installing package %1... + call :%1_features + REM vcpkg may not be reliable on slow, intermittent or proxy REM connections, see e.g. REM https://social.msdn.microsoft.com/Forums/windowsdesktop/en-US/4a8f7be5-5e15-4213-a7bb-ddf424a954e6/winhttpsendrequest-ends-with-12002-errorhttptimeout-after-21-seconds-no-matter-what-timeout?forum=windowssdk REM which explains the hidden 21 second timeout REM (last post by Dave : Microsoft - Windows Networking team) - .\vcpkg.exe install %1:%arch% + .\vcpkg.exe install %1%features%:%arch% IF ERRORLEVEL 1 ( EXIT /B 1 ) echo Finished %1 goto :EOF + +:: +:: features for each vcpkg to install +:: there should be an entry here for each package to install +:: 'set features=' means use the default otherwise +:: 'set features=[comma-delimited-feature-set]' is the syntax +:: + +:zlib_features +set features= +goto :EOF + +:expat_features +set features= +goto :EOF + +:libiconv_features +set features= +goto :EOF + +:openssl_features +set features= +goto :EOF + +:libssh2_features +set features= +goto :EOF + +:curl_features +set features=[core,openssl] +goto :EOF From 480cf60cc1b059b82929c5e517f3a94a655b51ef Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Fri, 2 Jul 2021 00:30:24 +0100 Subject: [PATCH 016/229] CMake: default Visual Studio generator has changed Correct some wording and inform users regarding the Visual Studio changes (from V16.6) to the default generator. Subsequent commits ensure that Git for Windows can be directly opened in modern Visual Studio without needing special configuration of the CMakeLists settings. It appeares that internally Visual Studio creates it's own version of the .sln file (etc.) for extension tools that expect them. The large number of references below document the shifting of Visual Studio default and CMake setting options. refs: https://docs.microsoft.com/en-us/search/?scope=C%2B%2B&view=msvc-150&terms=Ninja 1. https://docs.microsoft.com/en-us/cpp/linux/cmake-linux-configure?view=msvc-160 (note the linux bit) "In Visual Studio 2019 version 16.6 or later ***, Ninja is the default generator for configurations targeting a remote system or WSL. For more information, see this post on the C++ Team Blog [https://devblogs.microsoft.com/cppblog/linux-development-with-visual-studio-first-class-support-for-gdbserver-improved-build-times-with-ninja-and-updates-to-the-connection-manager/]. For more information about these settings, see CMakeSettings.json reference [https://docs.microsoft.com/en-us/cpp/build/cmakesettings-reference?view=msvc-160]." 2. https://docs.microsoft.com/en-us/cpp/build/cmake-presets-vs?view=msvc-160 "CMake supports two files that allow users to specify common configure, build, and test options and share them with others: CMakePresets.json and CMakeUserPresets.json." " Both files are supported in Visual Studio 2019 version 16.10 or later. ***" 3. https://devblogs.microsoft.com/cppblog/linux-development-with-visual-studio-first-class-support-for-gdbserver-improved-build-times-with-ninja-and-updates-to-the-connection-manager/ " Ninja has been the default generator (underlying build system) for CMake configurations targeting Windows for some time***, but in Visual Studio 2019 version 16.6 Preview 3*** we added support for Ninja on Linux." 4. https://docs.microsoft.com/en-us/cpp/build/cmakesettings-reference?view=msvc-160 " `generator`: specifies CMake generator to use for this configuration. May be one of: Visual Studio 2019 only: Visual Studio 16 2019 Visual Studio 16 2019 Win64 Visual Studio 16 2019 ARM Visual Studio 2017 and later: Visual Studio 15 2017 Visual Studio 15 2017 Win64 Visual Studio 15 2017 ARM Visual Studio 14 2015 Visual Studio 14 2015 Win64 Visual Studio 14 2015 ARM Unix Makefiles Ninja Because Ninja is designed for fast build speeds instead of flexibility and function, it is set as the default. However, some CMake projects may be unable to correctly build using Ninja. If this occurs, you can instruct CMake to generate Visual Studio projects instead. To specify a Visual Studio generator in Visual Studio 2017, open the settings editor from the main menu by choosing CMake | Change CMake Settings. Delete "Ninja" and type "V". This activates IntelliSense, which enables you to choose the generator you want." "To specify a Visual Studio generator in Visual Studio 2019, right-click on the CMakeLists.txt file in Solution Explorer and choose CMake Settings for project > Show Advanced Settings > CMake Generator. When the active configuration specifies a Visual Studio generator, by default MSBuild.exe is invoked with` -m -v:minimal` arguments." 5. https://docs.microsoft.com/en-us/cpp/build/cmake-presets-vs?view=msvc-160#enable-cmakepresetsjson-integration-in-visual-studio-2019 "Enable CMakePresets.json integration in Visual Studio 2019 CMakePresets.json integration isn't enabled by default in Visual Studio 2019. You can enable it for all CMake projects in Tools > Options > CMake > General: (tick a box)" ... see more. 6. https://docs.microsoft.com/en-us/cpp/build/cmakesettings-reference?view=msvc-140 (whichever v140 is..) "CMake projects are supported in Visual Studio 2017 and later." 7. https://docs.microsoft.com/en-us/cpp/overview/what-s-new-for-cpp-2017?view=msvc-150 "Support added for the CMake Ninja generator." 8. https://docs.microsoft.com/en-us/cpp/overview/what-s-new-for-cpp-2017?view=msvc-150#cmake-support-via-open-folder "CMake support via Open Folder Visual Studio 2017 introduces support for using CMake projects without converting to MSBuild project files (.vcxproj). For more information, see CMake projects in Visual Studio[https://docs.microsoft.com/en-us/cpp/build/cmake-projects-in-visual-studio?view=msvc-150]. Opening CMake projects with Open Folder automatically configures the environment for C++ editing, building, and debugging." ... +more! 9. https://docs.microsoft.com/en-us/cpp/build/cmake-presets-vs?view=msvc-160#supported-cmake-and-cmakepresetsjson-versions "Visual Studio reads and evaluates CMakePresets.json and CMakeUserPresets.json itself and doesn't invoke CMake directly with the --preset option. So, CMake version 3.20 or later isn't strictly required when you're building with CMakePresets.json inside Visual Studio. We recommend using CMake version 3.14 or later." 10. https://docs.microsoft.com/en-us/cpp/build/cmake-presets-vs?view=msvc-160#enable-cmakepresetsjson-integration-in-visual-studio-2019 "If you don't want to enable CMakePresets.json integration for all CMake projects, you can enable CMakePresets.json integration for a single CMake project by adding a CMakePresets.json file to the root of the open folder. You must close and reopen the folder in Visual Studio to activate the integration. 11. https://docs.microsoft.com/en-us/cpp/build/cmake-presets-vs?view=msvc-160#default-configure-presets ***(doesn't actually say which version..) "Default Configure Presets If no CMakePresets.json or CMakeUserPresets.json file exists, or if CMakePresets.json or CMakeUserPresets.json is invalid, Visual Studio will fall back*** on the following default Configure Presets: Windows example JSON { "name": "windows-default", "displayName": "Windows x64 Debug", "description": "Sets Ninja generator, compilers, x64 architecture, build and install directory, debug build type", "generator": "Ninja", "binaryDir": "${sourceDir}/out/build/${presetName}", "architecture": { "value": "x64", "strategy": "external" }, "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_INSTALL_PREFIX": "${sourceDir}/out/install/${presetName}" }, "vendor": { "microsoft.com/VisualStudioSettings/CMake/1.0": { "hostOS": [ "Windows" ] } } }, " Signed-off-by: Philip Oakley --- contrib/buildsystems/CMakeLists.txt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 5ce6fc48cc61ee..76d7022af3e6bb 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -14,6 +14,11 @@ Note: Visual Studio also has the option of opening `CMakeLists.txt` directly; Using this option, Visual Studio will not find the source code, though, therefore the `File>Open>Folder...` option is preferred. +Visual Studio does not produce a .sln solution file nor the .vcxproj files +that may be required by VS extension tools. + +To generate the .sln/.vcxproj files run CMake manually, as described below. + Instructions to run CMake manually: mkdir -p contrib/buildsystems/out @@ -22,7 +27,7 @@ Instructions to run CMake manually: This will build the git binaries in contrib/buildsystems/out directory (our top-level .gitignore file knows to ignore contents of -this directory). +this directory). The project .sln and .vcxproj files are also generated. Possible build configurations(-DCMAKE_BUILD_TYPE) with corresponding compiler flags @@ -35,17 +40,16 @@ empty(default) : NOTE: -DCMAKE_BUILD_TYPE is optional. For multi-config generators like Visual Studio this option is ignored -This process generates a Makefile(Linux/*BSD/MacOS) , Visual Studio solution(Windows) by default. +This process generates a Makefile(Linux/*BSD/MacOS), Visual Studio solution(Windows) by default. Run `make` to build Git on Linux/*BSD/MacOS. Open git.sln on Windows and build Git. -NOTE: By default CMake uses Makefile as the build tool on Linux and Visual Studio in Windows, -to use another tool say `ninja` add this to the command line when configuring. -`-G Ninja` - NOTE: By default CMake will install vcpkg locally to your source tree on configuration, to avoid this, add `-DNO_VCPKG=TRUE` to the command line when configuring. +The Visual Studio default generator changed in v16.6 from its Visual Studio +implemenation to `Ninja` This required changes to many CMake scripts. + ]] cmake_minimum_required(VERSION 3.14) From 042c55f1e81efa50dc66b926ec72a390809eb358 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 5 Apr 2021 14:12:14 -0400 Subject: [PATCH 017/229] config.mak.uname: add git.rc to MSVC builds Teach MSVC=1 builds to depend on the `git.rc` file so that the resulting executables have Windows-style resources and version number information within them. Signed-off-by: Jeff Hostetler --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 152ff78fe6d731..16582da35d8336 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -547,6 +547,7 @@ endif # handle twice, or to access the osfhandle of an already-closed stdout # See https://msdn.microsoft.com/en-us/library/ms235330.aspx EXTLIBS = user32.lib advapi32.lib shell32.lib wininet.lib ws2_32.lib invalidcontinue.obj kernel32.lib ntdll.lib + GITLIBS += git.res PTHREAD_LIBS = RC = compat/vcbuild/scripts/rc.pl lib = From 1ba3d12f7bd8867bc321dfa0ba8a666af2b949fb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 6 May 2023 22:26:15 +0200 Subject: [PATCH 018/229] http: optionally load libcurl lazily This compile-time option allows to ask Git to load libcurl dynamically at runtime. Together with a follow-up patch that optionally overrides the file name depending on the `http.sslBackend` setting, this kicks open the door for installing multiple libcurl flavors side by side, and load the one corresponding to the (runtime-)configured SSL/TLS backend. Signed-off-by: Johannes Schindelin --- Makefile | 28 +++- compat/lazyload-curl.c | 364 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 385 insertions(+), 7 deletions(-) create mode 100644 compat/lazyload-curl.c diff --git a/Makefile b/Makefile index 984559fd680f09..c9553f775744c4 100644 --- a/Makefile +++ b/Makefile @@ -481,6 +481,11 @@ include shared.mak # # CURL_LDFLAGS=-lcurl # +# Define LAZYLOAD_LIBCURL to dynamically load the libcurl; This can be useful +# if Multiple libcurl versions exist (with different file names) that link to +# various SSL/TLS backends, to support the `http.sslBackend` runtime switch in +# such a scenario. +# # === Optional library: libpcre2 === # # Define USE_LIBPCRE if you have and want to use libpcre. Various @@ -1816,10 +1821,19 @@ else CURL_LIBCURL = endif - ifndef CURL_LDFLAGS - CURL_LDFLAGS = $(eval CURL_LDFLAGS := $$(shell $$(CURL_CONFIG) --libs))$(CURL_LDFLAGS) + ifdef LAZYLOAD_LIBCURL + LAZYLOAD_LIBCURL_OBJ = compat/lazyload-curl.o + OBJECTS += $(LAZYLOAD_LIBCURL_OBJ) + # The `CURL_STATICLIB` constant must be defined to avoid seeing the functions + # declared as DLL imports + CURL_CFLAGS = -DCURL_STATICLIB + CURL_LIBCURL = -ldl + else + ifndef CURL_LDFLAGS + CURL_LDFLAGS = $(eval CURL_LDFLAGS := $$(shell $$(CURL_CONFIG) --libs))$(CURL_LDFLAGS) + endif + CURL_LIBCURL += $(CURL_LDFLAGS) endif - CURL_LIBCURL += $(CURL_LDFLAGS) ifndef CURL_CFLAGS CURL_CFLAGS = $(eval CURL_CFLAGS := $$(shell $$(CURL_CONFIG) --cflags))$(CURL_CFLAGS) @@ -1840,7 +1854,7 @@ else endif ifdef USE_CURL_FOR_IMAP_SEND BASIC_CFLAGS += -DUSE_CURL_FOR_IMAP_SEND - IMAP_SEND_BUILDDEPS = http.o + IMAP_SEND_BUILDDEPS = http.o $(LAZYLOAD_LIBCURL_OBJ) IMAP_SEND_LDFLAGS += $(CURL_LIBCURL) endif ifndef NO_EXPAT @@ -3038,10 +3052,10 @@ git-imap-send$X: imap-send.o $(IMAP_SEND_BUILDDEPS) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(IMAP_SEND_LDFLAGS) $(LIBS) -git-http-fetch$X: http.o http-walker.o http-fetch.o GIT-LDFLAGS $(GITLIBS) +git-http-fetch$X: http.o http-walker.o http-fetch.o $(LAZYLOAD_LIBCURL_OBJ) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(LIBS) -git-http-push$X: http.o http-push.o GIT-LDFLAGS $(GITLIBS) +git-http-push$X: http.o http-push.o $(LAZYLOAD_LIBCURL_OBJ) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) @@ -3051,7 +3065,7 @@ $(REMOTE_CURL_ALIASES): $(REMOTE_CURL_PRIMARY) ln -s $< $@ 2>/dev/null || \ cp $< $@ -$(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o GIT-LDFLAGS $(GITLIBS) +$(REMOTE_CURL_PRIMARY): remote-curl.o http.o http-walker.o $(LAZYLOAD_LIBCURL_OBJ) GIT-LDFLAGS $(GITLIBS) $(QUIET_LINK)$(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \ $(CURL_LIBCURL) $(EXPAT_LIBEXPAT) $(LIBS) diff --git a/compat/lazyload-curl.c b/compat/lazyload-curl.c new file mode 100644 index 00000000000000..f4e08f76dfcd7f --- /dev/null +++ b/compat/lazyload-curl.c @@ -0,0 +1,364 @@ +#include "../git-compat-util.h" +#include "../git-curl-compat.h" +#include + +/* + * The ABI version of libcurl is encoded in its shared libraries' file names. + * This ABI version has not changed since October 2006 and is unlikely to be + * changed in the future. See https://curl.se/libcurl/abi.html for details. + */ +#define LIBCURL_ABI_VERSION "4" + +typedef void (*func_t)(void); + +#ifdef __APPLE__ +#define LIBCURL_FILE_NAME(base) base "." LIBCURL_ABI_VERSION ".dylib" +#else +#define LIBCURL_FILE_NAME(base) base ".so." LIBCURL_ABI_VERSION +#endif + +static void *load_library(const char *name) +{ + return dlopen(name, RTLD_LAZY); +} + +static func_t load_function(void *handle, const char *name) +{ + /* + * Casting the return value of `dlsym()` to a function pointer is + * explicitly allowed in recent POSIX standards, but GCC complains + * about this in pedantic mode nevertheless. For more about this issue, + * see https://stackoverflow.com/q/31526876/1860823 and + * http://stackoverflow.com/a/36385690/1905491. + */ + func_t f; + *(void **)&f = dlsym(handle, name); + return f; +} + +typedef struct curl_version_info_data *(*curl_version_info_type)(CURLversion version); +static curl_version_info_type curl_version_info_func; + +typedef char *(*curl_easy_escape_type)(CURL *handle, const char *string, int length); +static curl_easy_escape_type curl_easy_escape_func; + +typedef void (*curl_free_type)(void *p); +static curl_free_type curl_free_func; + +typedef CURLcode (*curl_global_init_type)(long flags); +static curl_global_init_type curl_global_init_func; + +typedef CURLsslset (*curl_global_sslset_type)(curl_sslbackend id, const char *name, const curl_ssl_backend ***avail); +static curl_global_sslset_type curl_global_sslset_func; + +typedef void (*curl_global_cleanup_type)(void); +static curl_global_cleanup_type curl_global_cleanup_func; + +typedef CURLcode (*curl_global_trace_type)(const char *config); +static curl_global_trace_type curl_global_trace_func; + +typedef struct curl_slist *(*curl_slist_append_type)(struct curl_slist *list, const char *data); +static curl_slist_append_type curl_slist_append_func; + +typedef void (*curl_slist_free_all_type)(struct curl_slist *list); +static curl_slist_free_all_type curl_slist_free_all_func; + +typedef const char *(*curl_easy_strerror_type)(CURLcode error); +static curl_easy_strerror_type curl_easy_strerror_func; + +typedef CURLM *(*curl_multi_init_type)(void); +static curl_multi_init_type curl_multi_init_func; + +typedef CURLMcode (*curl_multi_add_handle_type)(CURLM *multi_handle, CURL *curl_handle); +static curl_multi_add_handle_type curl_multi_add_handle_func; + +typedef CURLMcode (*curl_multi_remove_handle_type)(CURLM *multi_handle, CURL *curl_handle); +static curl_multi_remove_handle_type curl_multi_remove_handle_func; + +typedef CURLMcode (*curl_multi_fdset_type)(CURLM *multi_handle, fd_set *read_fd_set, fd_set *write_fd_set, fd_set *exc_fd_set, int *max_fd); +static curl_multi_fdset_type curl_multi_fdset_func; + +typedef CURLMcode (*curl_multi_perform_type)(CURLM *multi_handle, int *running_handles); +static curl_multi_perform_type curl_multi_perform_func; + +typedef CURLMcode (*curl_multi_cleanup_type)(CURLM *multi_handle); +static curl_multi_cleanup_type curl_multi_cleanup_func; + +typedef CURLMsg *(*curl_multi_info_read_type)(CURLM *multi_handle, int *msgs_in_queue); +static curl_multi_info_read_type curl_multi_info_read_func; + +typedef const char *(*curl_multi_strerror_type)(CURLMcode error); +static curl_multi_strerror_type curl_multi_strerror_func; + +typedef CURLMcode (*curl_multi_timeout_type)(CURLM *multi_handle, long *milliseconds); +static curl_multi_timeout_type curl_multi_timeout_func; + +typedef CURL *(*curl_easy_init_type)(void); +static curl_easy_init_type curl_easy_init_func; + +typedef CURLcode (*curl_easy_perform_type)(CURL *curl); +static curl_easy_perform_type curl_easy_perform_func; + +typedef void (*curl_easy_cleanup_type)(CURL *curl); +static curl_easy_cleanup_type curl_easy_cleanup_func; + +typedef CURL *(*curl_easy_duphandle_type)(CURL *curl); +static curl_easy_duphandle_type curl_easy_duphandle_func; + +typedef CURLcode (*curl_easy_getinfo_long_type)(CURL *curl, CURLINFO info, long *value); +static curl_easy_getinfo_long_type curl_easy_getinfo_long_func; + +typedef CURLcode (*curl_easy_getinfo_pointer_type)(CURL *curl, CURLINFO info, void **value); +static curl_easy_getinfo_pointer_type curl_easy_getinfo_pointer_func; + +typedef CURLcode (*curl_easy_getinfo_off_t_type)(CURL *curl, CURLINFO info, curl_off_t *value); +static curl_easy_getinfo_off_t_type curl_easy_getinfo_off_t_func; + +typedef CURLcode (*curl_easy_setopt_long_type)(CURL *curl, CURLoption opt, long value); +static curl_easy_setopt_long_type curl_easy_setopt_long_func; + +typedef CURLcode (*curl_easy_setopt_pointer_type)(CURL *curl, CURLoption opt, void *value); +static curl_easy_setopt_pointer_type curl_easy_setopt_pointer_func; + +typedef CURLcode (*curl_easy_setopt_off_t_type)(CURL *curl, CURLoption opt, curl_off_t value); +static curl_easy_setopt_off_t_type curl_easy_setopt_off_t_func; + +static void lazy_load_curl(void) +{ + static int initialized; + void *libcurl; + func_t curl_easy_getinfo_func, curl_easy_setopt_func; + + if (initialized) + return; + + initialized = 1; + libcurl = load_library(LIBCURL_FILE_NAME("libcurl")); + if (!libcurl) + die("failed to load library '%s'", LIBCURL_FILE_NAME("libcurl")); + + curl_version_info_func = (curl_version_info_type)load_function(libcurl, "curl_version_info"); + curl_easy_escape_func = (curl_easy_escape_type)load_function(libcurl, "curl_easy_escape"); + curl_free_func = (curl_free_type)load_function(libcurl, "curl_free"); + curl_global_init_func = (curl_global_init_type)load_function(libcurl, "curl_global_init"); + curl_global_sslset_func = (curl_global_sslset_type)load_function(libcurl, "curl_global_sslset"); + curl_global_cleanup_func = (curl_global_cleanup_type)load_function(libcurl, "curl_global_cleanup"); + curl_global_trace_func = (curl_global_trace_type)load_function(libcurl, "curl_global_trace"); + curl_slist_append_func = (curl_slist_append_type)load_function(libcurl, "curl_slist_append"); + curl_slist_free_all_func = (curl_slist_free_all_type)load_function(libcurl, "curl_slist_free_all"); + curl_easy_strerror_func = (curl_easy_strerror_type)load_function(libcurl, "curl_easy_strerror"); + curl_multi_init_func = (curl_multi_init_type)load_function(libcurl, "curl_multi_init"); + curl_multi_add_handle_func = (curl_multi_add_handle_type)load_function(libcurl, "curl_multi_add_handle"); + curl_multi_remove_handle_func = (curl_multi_remove_handle_type)load_function(libcurl, "curl_multi_remove_handle"); + curl_multi_fdset_func = (curl_multi_fdset_type)load_function(libcurl, "curl_multi_fdset"); + curl_multi_perform_func = (curl_multi_perform_type)load_function(libcurl, "curl_multi_perform"); + curl_multi_cleanup_func = (curl_multi_cleanup_type)load_function(libcurl, "curl_multi_cleanup"); + curl_multi_info_read_func = (curl_multi_info_read_type)load_function(libcurl, "curl_multi_info_read"); + curl_multi_strerror_func = (curl_multi_strerror_type)load_function(libcurl, "curl_multi_strerror"); + curl_multi_timeout_func = (curl_multi_timeout_type)load_function(libcurl, "curl_multi_timeout"); + curl_easy_init_func = (curl_easy_init_type)load_function(libcurl, "curl_easy_init"); + curl_easy_perform_func = (curl_easy_perform_type)load_function(libcurl, "curl_easy_perform"); + curl_easy_cleanup_func = (curl_easy_cleanup_type)load_function(libcurl, "curl_easy_cleanup"); + curl_easy_duphandle_func = (curl_easy_duphandle_type)load_function(libcurl, "curl_easy_duphandle"); + + curl_easy_getinfo_func = load_function(libcurl, "curl_easy_getinfo"); + curl_easy_getinfo_long_func = (curl_easy_getinfo_long_type)curl_easy_getinfo_func; + curl_easy_getinfo_pointer_func = (curl_easy_getinfo_pointer_type)curl_easy_getinfo_func; + curl_easy_getinfo_off_t_func = (curl_easy_getinfo_off_t_type)curl_easy_getinfo_func; + + curl_easy_setopt_func = load_function(libcurl, "curl_easy_setopt"); + curl_easy_setopt_long_func = (curl_easy_setopt_long_type)curl_easy_setopt_func; + curl_easy_setopt_pointer_func = (curl_easy_setopt_pointer_type)curl_easy_setopt_func; + curl_easy_setopt_off_t_func = (curl_easy_setopt_off_t_type)curl_easy_setopt_func; +} + +struct curl_version_info_data *curl_version_info(CURLversion version) +{ + lazy_load_curl(); + return curl_version_info_func(version); +} + +char *curl_easy_escape(CURL *handle, const char *string, int length) +{ + lazy_load_curl(); + return curl_easy_escape_func(handle, string, length); +} + +void curl_free(void *p) +{ + lazy_load_curl(); + curl_free_func(p); +} + +CURLcode curl_global_init(long flags) +{ + lazy_load_curl(); + return curl_global_init_func(flags); +} + +CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, const curl_ssl_backend ***avail) +{ + lazy_load_curl(); + return curl_global_sslset_func(id, name, avail); +} + +void curl_global_cleanup(void) +{ + lazy_load_curl(); + curl_global_cleanup_func(); +} + +CURLcode curl_global_trace(const char *config) +{ + lazy_load_curl(); + return curl_global_trace_func(config); +} + +struct curl_slist *curl_slist_append(struct curl_slist *list, const char *data) +{ + lazy_load_curl(); + return curl_slist_append_func(list, data); +} + +void curl_slist_free_all(struct curl_slist *list) +{ + lazy_load_curl(); + curl_slist_free_all_func(list); +} + +const char *curl_easy_strerror(CURLcode error) +{ + lazy_load_curl(); + return curl_easy_strerror_func(error); +} + +CURLM *curl_multi_init(void) +{ + lazy_load_curl(); + return curl_multi_init_func(); +} + +CURLMcode curl_multi_add_handle(CURLM *multi_handle, CURL *curl_handle) +{ + lazy_load_curl(); + return curl_multi_add_handle_func(multi_handle, curl_handle); +} + +CURLMcode curl_multi_remove_handle(CURLM *multi_handle, CURL *curl_handle) +{ + lazy_load_curl(); + return curl_multi_remove_handle_func(multi_handle, curl_handle); +} + +CURLMcode curl_multi_fdset(CURLM *multi_handle, fd_set *read_fd_set, fd_set *write_fd_set, fd_set *exc_fd_set, int *max_fd) +{ + lazy_load_curl(); + return curl_multi_fdset_func(multi_handle, read_fd_set, write_fd_set, exc_fd_set, max_fd); +} + +CURLMcode curl_multi_perform(CURLM *multi_handle, int *running_handles) +{ + lazy_load_curl(); + return curl_multi_perform_func(multi_handle, running_handles); +} + +CURLMcode curl_multi_cleanup(CURLM *multi_handle) +{ + lazy_load_curl(); + return curl_multi_cleanup_func(multi_handle); +} + +CURLMsg *curl_multi_info_read(CURLM *multi_handle, int *msgs_in_queue) +{ + lazy_load_curl(); + return curl_multi_info_read_func(multi_handle, msgs_in_queue); +} + +const char *curl_multi_strerror(CURLMcode error) +{ + lazy_load_curl(); + return curl_multi_strerror_func(error); +} + +CURLMcode curl_multi_timeout(CURLM *multi_handle, long *milliseconds) +{ + lazy_load_curl(); + return curl_multi_timeout_func(multi_handle, milliseconds); +} + +CURL *curl_easy_init(void) +{ + lazy_load_curl(); + return curl_easy_init_func(); +} + +CURLcode curl_easy_perform(CURL *curl) +{ + lazy_load_curl(); + return curl_easy_perform_func(curl); +} + +void curl_easy_cleanup(CURL *curl) +{ + lazy_load_curl(); + curl_easy_cleanup_func(curl); +} + +CURL *curl_easy_duphandle(CURL *curl) +{ + lazy_load_curl(); + return curl_easy_duphandle_func(curl); +} + +#ifndef CURL_IGNORE_DEPRECATION +#define CURL_IGNORE_DEPRECATION(x) x +#endif + +#ifndef CURLOPTTYPE_BLOB +#define CURLOPTTYPE_BLOB 40000 +#endif + +#undef curl_easy_getinfo +CURLcode curl_easy_getinfo(CURL *curl, CURLINFO info, ...) +{ + va_list ap; + CURLcode res; + + va_start(ap, info); + lazy_load_curl(); + CURL_IGNORE_DEPRECATION( + if (info >= CURLINFO_LONG && info < CURLINFO_DOUBLE) + res = curl_easy_getinfo_long_func(curl, info, va_arg(ap, long *)); + else if ((info >= CURLINFO_STRING && info < CURLINFO_LONG) || + (info >= CURLINFO_SLIST && info < CURLINFO_SOCKET)) + res = curl_easy_getinfo_pointer_func(curl, info, va_arg(ap, void **)); + else if (info >= CURLINFO_OFF_T) + res = curl_easy_getinfo_off_t_func(curl, info, va_arg(ap, curl_off_t *)); + else + die("%s:%d: TODO (info: %d)!", __FILE__, __LINE__, info); + ) + va_end(ap); + return res; +} + +#undef curl_easy_setopt +CURLcode curl_easy_setopt(CURL *curl, CURLoption opt, ...) +{ + va_list ap; + CURLcode res; + + va_start(ap, opt); + lazy_load_curl(); + CURL_IGNORE_DEPRECATION( + if (opt >= CURLOPTTYPE_LONG && opt < CURLOPTTYPE_OBJECTPOINT) + res = curl_easy_setopt_long_func(curl, opt, va_arg(ap, long)); + else if (opt >= CURLOPTTYPE_OBJECTPOINT && opt < CURLOPTTYPE_OFF_T) + res = curl_easy_setopt_pointer_func(curl, opt, va_arg(ap, void *)); + else if (opt >= CURLOPTTYPE_OFF_T && opt < CURLOPTTYPE_BLOB) + res = curl_easy_setopt_off_t_func(curl, opt, va_arg(ap, curl_off_t)); + else + die("%s:%d: TODO (opt: %d)!", __FILE__, __LINE__, opt); + ) + va_end(ap); + return res; +} From f8c61a9ef8bfe98e07c859ade858febfa5614df7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 31 Jan 2020 12:02:47 +0100 Subject: [PATCH 019/229] mingw: demonstrate a `git add` issue with NTFS junctions NTFS junctions are somewhat similar in spirit to Unix bind mounts: they point to a different directory and are resolved by the filesystem driver. As such, they appear to `lstat()` as if they are directories, not as if they are symbolic links. _Any_ user can create junctions, while symbolic links can only be created by non-administrators in Developer Mode on Windows 10. Hence NTFS junctions are much more common "in the wild" than NTFS symbolic links. It was reported in https://github.com/git-for-windows/git/issues/2481 that adding files via an absolute path that traverses an NTFS junction: since 1e64d18 (mingw: do resolve symlinks in `getcwd()`), we resolve not only symbolic links but also NTFS junctions when determining the absolute path of the current directory. The same is not true for `git add `, where symbolic links are resolved in ``, but not NTFS junctions. Signed-off-by: Johannes Schindelin --- t/t3700-add.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/t/t3700-add.sh b/t/t3700-add.sh index 59e48482a28c76..3647569a4fc53c 100755 --- a/t/t3700-add.sh +++ b/t/t3700-add.sh @@ -587,4 +587,15 @@ test_expect_success CASE_INSENSITIVE_FS 'path is case-insensitive' ' git add "$downcased" ' +test_expect_failure MINGW 'can add files via NTFS junctions' ' + test_when_finished "cmd //c rmdir junction && rm -rf target" && + test_create_repo target && + cmd //c "mklink /j junction target" && + >target/via-junction && + git -C junction add "$(pwd)/junction/via-junction" && + echo via-junction >expect && + git -C target diff --cached --name-only >actual && + test_cmp expect actual +' + test_done From f946649cb2d513964c9a91be7a66751fedbc36c1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 10 Jun 2026 17:09:01 +0200 Subject: [PATCH 020/229] ci(vs-build): adapt to Visual Studio 2026 default on windows-latest The `windows-latest` runner image migration that began on June 8, 2026 and completes on June 15, 2026 switches the default Visual Studio install from VS 2022 (v17) to VS 2026 (v18), per https://github.com/actions/runner-images/issues/14017. CMake 4.x picks up the new generator name "Visual Studio 18 2026" automatically and, crucially, writes the solution file with the new `.slnx` (XML) extension rather than `.sln`. See https://github.com/Kitware/CMake/blob/v4.3.2/Source/cmGlobalVisualStudioGenerator.cxx#L1147-L1159 where `GetSLNFile()` appends an "x" to the filename when the generator version is `VS18` or newer. As a result, the `MSBuild` step in the `vs-build` job fails with MSBUILD : error MSB1009: Project file does not exist. Switch: git.sln because the file CMake actually wrote is `git.slnx`. An example of the failure can be seen at https://github.com/git-for-windows/git/actions/runs/27264770241/job/80556419519. Teach the step to prefer `git.slnx` and fall back to `git.sln` so that it works on both the new image and any runner still on VS 2022 during the week-long staggered rollout. The conditional is written in PowerShell rather than bash so the step stays on the default shell: `microsoft/setup-msbuild@v3` adds `msbuild` to the Windows `PATH` only, and an MSYS2 bash spawned by the SDK does not pick it up (an earlier attempt at this fix using `shell: bash` failed with `msbuild: command not found`, see https://github.com/git-for-windows/git/actions/runs/27290221733/job/80608493655). Letting MSBuild itself discover the solution by omitting the project argument is not an option here either: CMake emits all `*.vcxproj` files (one per `add_executable`/`add_library`, e.g. `git-daemon.vcxproj`, `common-main.vcxproj`, `ALL_BUILD.vcxproj`, ...) into the same build root as the solution file, and MSBuild's auto-discovery in `ProcessProjectSwitch()` (`dotnet/msbuild`, `src/MSBuild/XMake.cs`) rejects that combination as `AmbiguousProjectError` because it only disambiguates the special case of exactly two projects where one has a `.proj` extension. Additionally, drop the `-property:PlatformToolset=v142` argument that had been carried since 889cacb6 (ci: configure GitHub Actions for CI/PR, 2020-04-11), when this job was first configured for VS 2019. The VS 2026 install on `windows-latest` only ships the v144 toolset along with a v143 compatibility component (`Microsoft.VisualStudio.Component.VC.14.44.17.14.x86.x64`); v142 is no longer present, so the explicit pin would now also fail in its own right. Removing it lets MSBuild use whatever toolset CMake selected during configuration (v143 on a VS 2022 runner, v144 on a VS 2026 one), which keeps the configure and build steps consistent with each other regardless of which image picked up the job. Signed-off-by: Johannes Schindelin Assisted-by: Opus 4.7 --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b229739be8b5ec..623cfae32231c1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -222,7 +222,9 @@ jobs: cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON - name: MSBuild - run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 + run: | + $sln = if (Test-Path git.slnx) { 'git.slnx' } else { 'git.sln' } + msbuild $sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 - name: bundle artifact tar shell: bash env: From e23586419f44b46970c8685c65ef5c76dda7fc46 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Fri, 4 Dec 2020 14:11:34 +0100 Subject: [PATCH 021/229] cmake: allow building for Windows/ARM64 Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- contrib/buildsystems/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 9415c476ba4e85..77afcdfd09c061 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -65,9 +65,9 @@ if(USE_VCPKG) set(VCPKG_DIR "${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg") if(NOT EXISTS ${VCPKG_DIR}) message("Initializing vcpkg and building the Git's dependencies (this will take a while...)") - execute_process(COMMAND ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg_install.bat) + execute_process(COMMAND ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg_install.bat ${VCPKG_ARCH}) endif() - list(APPEND CMAKE_PREFIX_PATH "${VCPKG_DIR}/installed/x64-windows") + list(APPEND CMAKE_PREFIX_PATH "${VCPKG_DIR}/installed/${VCPKG_ARCH}") # In the vcpkg edition, we need this to be able to link to libcurl set(CURL_NO_CURL_CMAKE ON) @@ -1216,7 +1216,7 @@ string(REPLACE "@USE_LIBPCRE2@" "" git_build_options "${git_build_options}") string(REPLACE "@WITH_BREAKING_CHANGES@" "" git_build_options "${git_build_options}") string(REPLACE "@X@" "${EXE_EXTENSION}" git_build_options "${git_build_options}") if(USE_VCPKG) - string(APPEND git_build_options "PATH=\"$PATH:$TEST_DIRECTORY/../compat/vcbuild/vcpkg/installed/x64-windows/bin\"\n") + string(APPEND git_build_options "PATH=\"$PATH:$TEST_DIRECTORY/../compat/vcbuild/vcpkg/installed/${VCPKG_ARCH}/bin\"\n") endif() file(WRITE ${CMAKE_BINARY_DIR}/GIT-BUILD-OPTIONS ${git_build_options}) From a2785303f25dab0f06a2aa7620f62a18f722203a Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sat, 24 Apr 2021 11:09:58 +0100 Subject: [PATCH 022/229] .gitignore: add Visual Studio CMakeSetting.json file The CMakeSettings.json file is tool generated. Developers may track it should they provide additional settings. Signed-off-by: Philip Oakley --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6b3f79165c9dab..aa0f2ea2113ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -259,6 +259,7 @@ Release/ /git.VC.db *.dSYM /contrib/buildsystems/out +CMakeSettings.json /contrib/libgit-rs/target /contrib/libgit-sys/target /rust/target From d7fb64fec06e61fd579bc3e772edd6f286f7c268 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 5 Apr 2021 14:24:52 -0400 Subject: [PATCH 023/229] clink.pl: ignore no-stack-protector arg on MSVC=1 builds Ignore the `-fno-stack-protector` compiler argument when building with MSVC. This will be used in a later commit that needs to build a Win32 GUI app. Signed-off-by: Jeff Hostetler --- compat/vcbuild/scripts/clink.pl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 2768ae15f1879f..73c8a2b184f38b 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -122,6 +122,8 @@ push(@cflags, "-wd4996"); } elsif ("$arg" =~ /^-W[a-z]/) { # let's ignore those + } elsif ("$arg" eq "-fno-stack-protector") { + # eat this } else { push(@args, $arg); } From a405c0ea2b391822bd66ffc3ff53bcfba80c3bea Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 7 May 2023 22:51:52 +0200 Subject: [PATCH 024/229] http: support lazy-loading libcurl also on Windows This implements the Windows-specific support code, because everything is slightly different on Windows, even loading shared libraries. Note: I specifically do _not_ use the code from `compat/win32/lazyload.h` here because that code is optimized for loading individual functions from various system DLLs, while we specifically want to load _many_ functions from _one_ DLL here, and distinctly not a system DLL (we expect libcurl to be located outside `C:\Windows\system32`, something `INIT_PROC_ADDR` refuses to work with). Also, the `curl_easy_getinfo()`/`curl_easy_setopt()` functions are declared as vararg functions, which `lazyload.h` cannot handle. Finally, we are about to optionally override the exact file name that is to be loaded, which is a goal contrary to `lazyload.h`'s design. Signed-off-by: Johannes Schindelin --- Makefile | 4 ++++ compat/lazyload-curl.c | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/Makefile b/Makefile index c9553f775744c4..8803ce526154ce 100644 --- a/Makefile +++ b/Makefile @@ -1827,7 +1827,11 @@ else # The `CURL_STATICLIB` constant must be defined to avoid seeing the functions # declared as DLL imports CURL_CFLAGS = -DCURL_STATICLIB +ifneq ($(uname_S),MINGW) +ifneq ($(uname_S),Windows) CURL_LIBCURL = -ldl +endif +endif else ifndef CURL_LDFLAGS CURL_LDFLAGS = $(eval CURL_LDFLAGS := $$(shell $$(CURL_CONFIG) --libs))$(CURL_LDFLAGS) diff --git a/compat/lazyload-curl.c b/compat/lazyload-curl.c index f4e08f76dfcd7f..82ab11de43a0fb 100644 --- a/compat/lazyload-curl.c +++ b/compat/lazyload-curl.c @@ -1,6 +1,8 @@ #include "../git-compat-util.h" #include "../git-curl-compat.h" +#ifndef WIN32 #include +#endif /* * The ABI version of libcurl is encoded in its shared libraries' file names. @@ -11,6 +13,7 @@ typedef void (*func_t)(void); +#ifndef WIN32 #ifdef __APPLE__ #define LIBCURL_FILE_NAME(base) base "." LIBCURL_ABI_VERSION ".dylib" #else @@ -35,6 +38,55 @@ static func_t load_function(void *handle, const char *name) *(void **)&f = dlsym(handle, name); return f; } +#else +#define LIBCURL_FILE_NAME(base) base "-" LIBCURL_ABI_VERSION ".dll" + +static void *load_library(const char *name) +{ + size_t name_size = strlen(name) + 1; + const char *path = getenv("PATH"); + char dll_path[MAX_PATH]; + + while (path && *path) { + const char *sep = strchrnul(path, ';'); + size_t len = sep - path; + + if (len && len + name_size < sizeof(dll_path)) { + memcpy(dll_path, path, len); + dll_path[len] = '/'; + memcpy(dll_path + len + 1, name, name_size); + + if (!access(dll_path, R_OK)) { + wchar_t wpath[MAX_PATH]; + int wlen = MultiByteToWideChar(CP_UTF8, 0, dll_path, -1, wpath, ARRAY_SIZE(wpath)); + void *res = wlen ? (void *)LoadLibraryExW(wpath, NULL, 0) : NULL; + if (!res) { + DWORD err = GetLastError(); + char buf[1024]; + + if (!FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM | + FORMAT_MESSAGE_ARGUMENT_ARRAY | + FORMAT_MESSAGE_IGNORE_INSERTS, + NULL, err, LANG_NEUTRAL, + buf, sizeof(buf) - 1, NULL)) + xsnprintf(buf, sizeof(buf), "last error: %ld", err); + error("LoadLibraryExW() failed with: %s", buf); + } + return res; + } + } + + path = *sep ? sep + 1 : NULL; + } + + return NULL; +} + +static func_t load_function(void *handle, const char *name) +{ + return (func_t)GetProcAddress((HANDLE)handle, name); +} +#endif typedef struct curl_version_info_data *(*curl_version_info_type)(CURLversion version); static curl_version_info_type curl_version_info_func; From 066c08475aa9b9f1fd44f3bdb84173684ea6e8fa Mon Sep 17 00:00:00 2001 From: Sverre Rabbelier Date: Sun, 24 Jul 2011 00:06:00 +0200 Subject: [PATCH 025/229] remote-helper: check helper status after import/export Signed-off-by: Johannes Schindelin Signed-off-by: Sverre Rabbelier --- t/t5801-remote-helpers.sh | 2 +- transport-helper.c | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/t/t5801-remote-helpers.sh b/t/t5801-remote-helpers.sh index 5fafc857b85627..ca4373d10db0c8 100755 --- a/t/t5801-remote-helpers.sh +++ b/t/t5801-remote-helpers.sh @@ -262,7 +262,7 @@ test_expect_success 'push update refs failure' ' echo "update fail" >>file && git commit -a -m "update fail" && git rev-parse --verify testgit/origin/heads/update >expect && - test_expect_code 1 env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ + test_must_fail env GIT_REMOTE_TESTGIT_FAILURE="non-fast forward" \ git push origin update && git rev-parse --verify testgit/origin/heads/update >actual && test_cmp expect actual diff --git a/transport-helper.c b/transport-helper.c index 4dda7df7de9aef..9a35343a8782fd 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -507,6 +507,19 @@ static int get_exporter(struct transport *transport, return start_command(fastexport); } +static void check_helper_status(struct helper_data *data) +{ + int pid, status; + + pid = waitpid(data->helper->pid, &status, WNOHANG); + if (pid < 0) + die("Could not retrieve status of remote helper '%s'", + data->name); + if (pid > 0 && WIFEXITED(status)) + die("Remote helper '%s' died with %d", + data->name, WEXITSTATUS(status)); +} + static int fetch_with_import(struct transport *transport, int nr_heads, struct ref **to_fetch) { @@ -543,6 +556,7 @@ static int fetch_with_import(struct transport *transport, if (finish_command(&fastimport)) die(_("error while running fast-import")); + check_helper_status(data); /* * The fast-import stream of a remote helper that advertises @@ -1198,6 +1212,7 @@ static int push_refs_with_export(struct transport *transport, if (finish_command(&exporter)) die(_("error while running fast-export")); + check_helper_status(data); if (push_update_refs_status(data, remote_refs, flags)) return 1; From fa91602ccf05f2bf1befc76e7934817e9f2f7e1b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 31 Jan 2020 11:44:31 +0100 Subject: [PATCH 026/229] strbuf_realpath(): use platform-dependent API if available Some platforms (e.g. Windows) provide API functions to resolve paths much quicker. Let's offer a way to short-cut `strbuf_realpath()` on those platforms. Signed-off-by: Johannes Schindelin --- abspath.c | 3 +++ git-compat-util.h | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/abspath.c b/abspath.c index 1202cde23dbc9b..0c17e98654e4b0 100644 --- a/abspath.c +++ b/abspath.c @@ -93,6 +93,9 @@ static char *strbuf_realpath_1(struct strbuf *resolved, const char *path, goto error_out; } + if (platform_strbuf_realpath(resolved, path)) + return resolved->buf; + strbuf_addstr(&remaining, path); get_root_part(resolved, &remaining); diff --git a/git-compat-util.h b/git-compat-util.h index a0f901ce793ed6..3240443ae439fe 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -354,6 +354,10 @@ static inline int git_has_dir_sep(const char *path) #define query_user_email() NULL #endif +#ifndef platform_strbuf_realpath +#define platform_strbuf_realpath(resolved, path) NULL +#endif + #ifdef __TANDEM #include #include From 08c5d138380011f39646dac9172bd563ced37241 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 9 May 2020 16:19:06 +0200 Subject: [PATCH 027/229] t5505/t5516: allow running without `.git/branches/` in the templates When we commit the template directory as part of `make vcxproj`, the `branches/` directory is not actually commited, as it is empty. Two tests were not prepared for that situation. This developer tried to get rid of the support for `.git/branches/` a long time ago, but that effort did not bear fruit, so the best we can do is work around in these here tests. Signed-off-by: Johannes Schindelin --- t/t5505-remote.sh | 4 ++-- t/t5516-fetch-push.sh | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh index 5fbfcb084830f2..d8af3c980008f9 100755 --- a/t/t5505-remote.sh +++ b/t/t5505-remote.sh @@ -1299,7 +1299,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'migrate a remote from named file in ( cd six && git remote rm origin && - mkdir .git/branches && + mkdir -p .git/branches && echo "$origin_url#main" >.git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && @@ -1314,7 +1314,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'migrate a remote from named file in ( cd seven && git remote rm origin && - mkdir .git/branches && + mkdir -p .git/branches && echo "quux#foom" > .git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index b982b209bfd51e..fcb039dd6659da 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -963,7 +963,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'fetch with branches' ' mk_empty testrepo && git branch second $the_first_commit && git checkout second && - mkdir testrepo/.git/branches && + mkdir -p testrepo/.git/branches && echo ".." > testrepo/.git/branches/branch1 && ( cd testrepo && @@ -977,7 +977,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'fetch with branches' ' test_expect_success !WITH_BREAKING_CHANGES 'fetch with branches containing #' ' mk_empty testrepo && - mkdir testrepo/.git/branches && + mkdir -p testrepo/.git/branches && echo "..#second" > testrepo/.git/branches/branch2 && ( cd testrepo && @@ -994,7 +994,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'push with branches' ' git checkout second && test_when_finished "rm -rf .git/branches" && - mkdir .git/branches && + mkdir -p .git/branches && echo "testrepo" > .git/branches/branch1 && git push branch1 && @@ -1010,7 +1010,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'push with branches containing #' ' mk_empty testrepo && test_when_finished "rm -rf .git/branches" && - mkdir .git/branches && + mkdir -p .git/branches && echo "testrepo#branch3" > .git/branches/branch2 && git push branch2 && From a28e118f44502fed582f5430df76df5ee0730d4b Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 6 Oct 2019 18:40:55 +0100 Subject: [PATCH 028/229] vcpkg_install: detect lack of Git The vcpkg_install batch file depends on the availability of a working Git on the CMD path. This may not be present if the user has selected the 'bash only' option during Git-for-Windows install. Detect and tell the user about their lack of a working Git in the CMD window. Fixes #2348. A separate PR https://github.com/git-for-windows/build-extra/pull/258 now highlights the recommended path setting during install. Signed-off-by: Philip Oakley --- compat/vcbuild/vcpkg_install.bat | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index ebd0bad242a8ca..bcbbf536af3141 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -36,6 +36,13 @@ REM ================================================================ dir vcpkg\vcpkg.exe >nul 2>nul && GOTO :install_libraries + git.exe version 2>nul + IF ERRORLEVEL 1 ( + echo "***" + echo "Git not found. Please adjust your CMD path or Git install option." + echo "***" + EXIT /B 1 ) + echo Fetching vcpkg in %cwd%vcpkg git.exe clone https://github.com/Microsoft/vcpkg vcpkg IF ERRORLEVEL 1 ( EXIT /B 1 ) From c977e07bc57e76ae5041397773a8537d8850a490 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 29 Nov 2020 00:12:26 +0100 Subject: [PATCH 029/229] ci(vs-build) also build Windows/ARM64 artifacts There are no Windows/ARM64 agents in GitHub Actions yet, therefore we just skip adjusting the `vs-test` job for now. Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 623cfae32231c1..94324e4829ea5c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -195,8 +195,11 @@ jobs: NO_PERL: 1 GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" runs-on: windows-latest + strategy: + matrix: + arch: [x64, arm64] concurrency: - group: vs-build-${{ github.ref }} + group: vs-build-${{ github.ref }}-${{ matrix.arch }} cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - uses: actions/checkout@v6 @@ -215,16 +218,16 @@ jobs: uses: microsoft/setup-msbuild@v3 - name: copy dlls to root shell: cmd - run: compat\vcbuild\vcpkg_copy_dlls.bat release + run: compat\vcbuild\vcpkg_copy_dlls.bat release ${{ matrix.arch }}-windows - name: generate Visual Studio solution shell: bash run: | - cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ - -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON + cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/${{ matrix.arch }}-windows \ + -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=${{ matrix.arch }} -DVCPKG_ARCH=${{ matrix.arch }}-windows - name: MSBuild run: | $sln = if (Test-Path git.slnx) { 'git.slnx' } else { 'git.sln' } - msbuild $sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 + msbuild $sln -property:Configuration=Release -property:Platform=${{ matrix.arch }} -maxCpuCount:4 - name: bundle artifact tar shell: bash env: @@ -238,7 +241,7 @@ jobs: - name: upload tracked files and build artifacts uses: actions/upload-artifact@v7 with: - name: vs-artifacts + name: vs-artifacts-${{ matrix.arch }} path: artifacts vs-test: name: win+VS test @@ -256,7 +259,7 @@ jobs: - name: download tracked files and build artifacts uses: actions/download-artifact@v8 with: - name: vs-artifacts + name: vs-artifacts-x64 path: ${{github.workspace}} - name: extract tracked files and build artifacts shell: bash From 1987e0c26eabeee8eb82966bfff8c52279013baa Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Thu, 5 Aug 2021 19:04:13 -0400 Subject: [PATCH 030/229] subtree: update `contrib/subtree` `test` target The intention of this change is to align with how the top-level git `Makefile` defines its own test target (which also internally calls `$(MAKE) -C t/ all`). This change also ensures the consistency of `make -C contrib/subtree test` with other testing in CI executions (which rely on `$DEFAULT_TEST_TARGET` being defined as `prove`). Signed-off-by: Victoria Dye --- contrib/subtree/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/subtree/Makefile b/contrib/subtree/Makefile index c0c9f21cb78022..dab2dfc08ee222 100644 --- a/contrib/subtree/Makefile +++ b/contrib/subtree/Makefile @@ -95,7 +95,7 @@ $(GIT_SUBTREE_TEST): $(GIT_SUBTREE) cp $< $@ test: $(GIT_SUBTREE_TEST) - $(MAKE) -C t/ test + $(MAKE) -C t/ all clean: $(RM) $(GIT_SUBTREE) From 5c356bab9d01ef7cf52c7e5e1a160802b6cd9308 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Thu, 22 Apr 2021 11:11:38 +0100 Subject: [PATCH 031/229] CMakeLists: add default "x64-windows" arch for Visual Studio In Git-for-Windows, work on using ARM64 has progressed. The commit 2d94b77b27 (cmake: allow building for Windows/ARM64, 2020-12-04) failed to notice that /compat/vcbuild/vcpkg_install.bat will default to using the "x64-windows" architecture for the vcpkg installation if not set, but CMake is not told of this default. Commit 635b6d99b3 (vcbuild: install ARM64 dependencies when building ARM64 binaries, 2020-01-31) later updated vcpkg_install.bat to accept an arch (%1) parameter, but retained the default. This default is neccessary for the use case where the project directory is opened directly in Visual Studio, which will find and build a CMakeLists.txt file without any parameters, thus expecting use of the default setting. Also Visual studio will generate internal .sln solution and .vcxproj project files needed for some extension tools. Inform users of the additional .sln/.vcxproj generation. ** How to test: rm -rf '.vs' # remove old visual studio settings rm -rf 'compat/vcbuild/vcpkg' # remove any vcpkg downloads rm -rf 'contrib/buildsystems/out' # remove builds & CMake artifacts with a fresh Visual Studio Community Edition, File>>Open>>(git *folder*) to load the project (which will take some time!). check for successful compilation. The implicit .sln (etc.) are in the hidden .vs directory created by Visual Studio. Signed-off-by: Philip Oakley --- contrib/buildsystems/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 76d7022af3e6bb..a68f44d5f8f88a 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -71,6 +71,10 @@ if(USE_VCPKG) message("Initializing vcpkg and building the Git's dependencies (this will take a while...)") execute_process(COMMAND ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg_install.bat ${VCPKG_ARCH}) endif() + if(NOT EXISTS ${VCPKG_ARCH}) + message("VCPKG_ARCH: unset, using 'x64-windows'") + set(VCPKG_ARCH "x64-windows") # default from vcpkg_install.bat + endif() list(APPEND CMAKE_PREFIX_PATH "${VCPKG_DIR}/installed/${VCPKG_ARCH}") # In the vcpkg edition, we need this to be able to link to libcurl From 8fd658e06b15dfab88c0bc264dbc766d67e4cd6d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 13 Apr 2022 14:49:17 -0400 Subject: [PATCH 032/229] setup: properly use "%(prefix)/" when in WSL Signed-off-by: Derrick Stolee --- setup.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/setup.c b/setup.c index 61bcdafe82a20b..7fce5833338d82 100644 --- a/setup.c +++ b/setup.c @@ -2006,10 +2006,19 @@ static void repo_discover(struct repo_discovery *discovery, int *nongit_ok) break; case GIT_DIR_INVALID_OWNERSHIP: if (!nongit_ok) { + struct strbuf prequoted = STRBUF_INIT; struct strbuf quoted = STRBUF_INIT; strbuf_complete(&report, '\n'); - sq_quote_buf_pretty("ed, dir.buf); + +#ifdef __MINGW32__ + if (dir.buf[0] == '/') + strbuf_addstr(&prequoted, "%(prefix)/"); +#endif + + strbuf_add(&prequoted, dir.buf, dir.len); + sq_quote_buf_pretty("ed, prequoted.buf); + die(_("detected dubious ownership in repository at '%s'\n" "%s" "To add an exception for this directory, call:\n" From 53ea655663e06de401ae97d0e3aa4f8383c81380 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 5 Apr 2021 14:39:33 -0400 Subject: [PATCH 033/229] clink.pl: move default linker options for MSVC=1 builds Move the default `-ENTRY` and `-SUBSYSTEM` arguments for MSVC=1 builds from `config.mak.uname` into `clink.pl`. These args are constant for console-mode executables. Add support to `clink.pl` for generating a Win32 GUI application using the `-mwindows` argument (to match how GCC does it). This changes the `-ENTRY` and `-SUBSYSTEM` arguments accordingly. Signed-off-by: Jeff Hostetler --- compat/vcbuild/scripts/clink.pl | 11 +++++++++++ config.mak.uname | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 73c8a2b184f38b..a38b360015ece9 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -15,6 +15,7 @@ my @lflags = (); my $is_linking = 0; my $is_debug = 0; +my $is_gui = 0; while (@ARGV) { my $arg = shift @ARGV; if ("$arg" eq "-DDEBUG") { @@ -124,11 +125,21 @@ # let's ignore those } elsif ("$arg" eq "-fno-stack-protector") { # eat this + } elsif ("$arg" eq "-mwindows") { + $is_gui = 1; } else { push(@args, $arg); } } if ($is_linking) { + if ($is_gui) { + push(@args, "-ENTRY:wWinMainCRTStartup"); + push(@args, "-SUBSYSTEM:WINDOWS"); + } else { + push(@args, "-ENTRY:wmainCRTStartup"); + push(@args, "-SUBSYSTEM:CONSOLE"); + } + push(@args, @lflags); unshift(@args, "link.exe"); } else { diff --git a/config.mak.uname b/config.mak.uname index 16582da35d8336..4d5c8d321ed59c 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -542,7 +542,7 @@ endif COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY \ -DENSURE_MSYSTEM_IS_SET="\"$(MSYSTEM)\"" -DMINGW_PREFIX="\"$(patsubst /%,%,$(MINGW_PREFIX))\"" \ -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" - BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO -ENTRY:wmainCRTStartup -SUBSYSTEM:CONSOLE + BASIC_LDFLAGS = -IGNORE:4217 -IGNORE:4049 -NOLOGO # invalidcontinue.obj allows Git's source code to close the same file # handle twice, or to access the osfhandle of an already-closed stdout # See https://msdn.microsoft.com/en-us/library/ms235330.aspx From 112240e87e9231953faced51ec367f281712606c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 7 May 2023 22:05:33 +0200 Subject: [PATCH 034/229] http: when loading libcurl lazily, allow for multiple SSL backends The previous commits introduced a compile-time option to load libcurl lazily, but it uses the hard-coded name "libcurl-4.dll" (or equivalent on platforms other than Windows). To allow for installing multiple libcurl flavors side by side, where each supports one specific SSL/TLS backend, let's first look whether `libcurl--4.dll` exists, and only use `libcurl-4.dll` as a fall back. That will allow us to ship with a libcurl by default that only supports the Secure Channel backend for the `https://` protocol. This libcurl won't suffer from any dependency problem when upgrading OpenSSL to a new major version (which will change the DLL name, and hence break every program and library that depends on it). This is crucial because Git for Windows relies on libcurl to keep working when building and deploying a new OpenSSL package because that library is used by `git fetch` and `git clone`. Note that this feature is by no means specific to Windows. On Ubuntu, for example, a `git` built using `LAZY_LOAD_LIBCURL` will use `libcurl.so.4` for `http.sslbackend=openssl` and `libcurl-gnutls.so.4` for `http.sslbackend=gnutls`. Signed-off-by: Johannes Schindelin --- compat/lazyload-curl.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/compat/lazyload-curl.c b/compat/lazyload-curl.c index 82ab11de43a0fb..a6a3f7e3a7aeaa 100644 --- a/compat/lazyload-curl.c +++ b/compat/lazyload-curl.c @@ -175,17 +175,26 @@ static curl_easy_setopt_pointer_type curl_easy_setopt_pointer_func; typedef CURLcode (*curl_easy_setopt_off_t_type)(CURL *curl, CURLoption opt, curl_off_t value); static curl_easy_setopt_off_t_type curl_easy_setopt_off_t_func; +static char ssl_backend[64]; + static void lazy_load_curl(void) { static int initialized; - void *libcurl; + void *libcurl = NULL; func_t curl_easy_getinfo_func, curl_easy_setopt_func; if (initialized) return; initialized = 1; - libcurl = load_library(LIBCURL_FILE_NAME("libcurl")); + if (ssl_backend[0]) { + char dll_name[64 + 16]; + snprintf(dll_name, sizeof(dll_name) - 1, + LIBCURL_FILE_NAME("libcurl-%s"), ssl_backend); + libcurl = load_library(dll_name); + } + if (!libcurl) + libcurl = load_library(LIBCURL_FILE_NAME("libcurl")); if (!libcurl) die("failed to load library '%s'", LIBCURL_FILE_NAME("libcurl")); @@ -250,6 +259,9 @@ CURLcode curl_global_init(long flags) CURLsslset curl_global_sslset(curl_sslbackend id, const char *name, const curl_ssl_backend ***avail) { + if (name && strlen(name) < sizeof(ssl_backend)) + strlcpy(ssl_backend, name, sizeof(ssl_backend)); + lazy_load_curl(); return curl_global_sslset_func(id, name, avail); } From aa2aea215a6265f2a2d4f4527dc402dcab2e9084 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 9 Apr 2012 13:04:35 -0500 Subject: [PATCH 035/229] Always auto-gc after calling a fast-import transport After importing anything with fast-import, we should always let the garbage collector do its job, since the objects are written to disk inefficiently. This brings down an initial import of http://selenic.com/hg from about 230 megabytes to about 14. In the future, we may want to make this configurable on a per-remote basis, or maybe teach fast-import about it in the first place. Signed-off-by: Johannes Schindelin --- transport-helper.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/transport-helper.c b/transport-helper.c index 9a35343a8782fd..c9d05075c316f6 100644 --- a/transport-helper.c +++ b/transport-helper.c @@ -22,6 +22,8 @@ #include "packfile.h" static int debug; +/* TODO: put somewhere sensible, e.g. git_transport_options? */ +static int auto_gc = 1; struct helper_data { char *name; @@ -590,6 +592,13 @@ static int fetch_with_import(struct transport *transport, } } strbuf_release(&buf); + if (auto_gc) { + struct child_process cmd = CHILD_PROCESS_INIT; + + cmd.git_cmd = 1; + strvec_pushl(&cmd.args, "gc", "--auto", "--quiet", NULL); + run_command(&cmd); + } return 0; } From ba01b8e0dbf6a371e98e304b199f22cc98c15540 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 Apr 2017 12:09:08 +0200 Subject: [PATCH 036/229] mingw: prevent regressions with "drive-less" absolute paths On Windows, there are several categories of absolute paths. One such category starts with a backslash and is implicitly relative to the drive associated with the current working directory. Example: c: git clone https://github.com/git-for-windows/git \G4W should clone into C:\G4W. Back in 2017, Juan Carlos Arevalo Baeza reported a bug in Git's handling of those absolute paths was identified, and fixed. Let's make sure that it stays fixed. Signed-off-by: Johannes Schindelin --- t/t5580-unc-paths.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/t/t5580-unc-paths.sh b/t/t5580-unc-paths.sh index 65ef1a3628ee94..e9df367d5777fd 100755 --- a/t/t5580-unc-paths.sh +++ b/t/t5580-unc-paths.sh @@ -20,14 +20,11 @@ fi UNCPATH="$(winpwd)" case "$UNCPATH" in [A-Z]:*) + WITHOUTDRIVE="${UNCPATH#?:}" # Use administrative share e.g. \\localhost\C$\git-sdk-64\usr\src\git # (we use forward slashes here because MSYS2 and Git accept them, and # they are easier on the eyes) - UNCPATH="//localhost/${UNCPATH%%:*}\$/${UNCPATH#?:}" - test -d "$UNCPATH" || { - skip_all='could not access administrative share; skipping' - test_done - } + UNCPATH="//localhost/${UNCPATH%%:*}\$$WITHOUTDRIVE" ;; *) skip_all='skipping UNC path tests, cannot determine current path as UNC' @@ -35,6 +32,18 @@ case "$UNCPATH" in ;; esac +test_expect_success 'clone into absolute path lacking a drive prefix' ' + USINGBACKSLASHES="$(echo "$WITHOUTDRIVE"/without-drive-prefix | + tr / \\\\)" && + git clone . "$USINGBACKSLASHES" && + test -f without-drive-prefix/.git/HEAD +' + +test -d "$UNCPATH" || { + skip_all='could not access administrative share; skipping' + test_done +} + test_expect_success setup ' test_commit initial ' From 7bf91c96ade23780304d803823bce3a0c56eadc8 Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Thu, 8 May 2014 21:43:24 +0200 Subject: [PATCH 037/229] transport: optionally disable side-band-64k Since commit 0c499ea60fda (send-pack: demultiplex a sideband stream with status data, 2010-02-05) the send-pack builtin uses the side-band-64k capability if advertised by the server. Unfortunately this breaks pushing over the dump git protocol if used over a network connection. The detailed reasons for this breakage are (by courtesy of Jeff Preshing, quoted from https://groups.google.com/d/msg/msysgit/at8D7J-h7mw/eaLujILGUWoJ): MinGW wraps Windows sockets in CRT file descriptors in order to mimic the functionality of POSIX sockets. This causes msvcrt.dll to treat sockets as Installable File System (IFS) handles, calling ReadFile, WriteFile, DuplicateHandle and CloseHandle on them. This approach works well in simple cases on recent versions of Windows, but does not support all usage patterns. In particular, using this approach, any attempt to read & write concurrently on the same socket (from one or more processes) will deadlock in a scenario where the read waits for a response from the server which is only invoked after the write. This is what send_pack currently attempts to do in the use_sideband codepath. The new config option `sendpack.sideband` allows to override the side-band-64k capability of the server, and thus makes the dumb git protocol work. Other transportation methods like ssh and http/https still benefit from the sideband channel, therefore the default value of `sendpack.sideband` is still true. Signed-off-by: Thomas Braun Signed-off-by: Oliver Schneider Signed-off-by: Johannes Schindelin --- Documentation/config.adoc | 2 ++ Documentation/config/sendpack.adoc | 5 +++++ send-pack.c | 6 +++--- 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 Documentation/config/sendpack.adoc diff --git a/Documentation/config.adoc b/Documentation/config.adoc index ceb13ade6e4fa2..65c7439d40fdcc 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -580,6 +580,8 @@ include::config/safe.adoc[] include::config/sendemail.adoc[] +include::config/sendpack.adoc[] + include::config/sequencer.adoc[] include::config/showbranch.adoc[] diff --git a/Documentation/config/sendpack.adoc b/Documentation/config/sendpack.adoc new file mode 100644 index 00000000000000..e306f657fba7dd --- /dev/null +++ b/Documentation/config/sendpack.adoc @@ -0,0 +1,5 @@ +sendpack.sideband:: + Allows to disable the side-band-64k capability for send-pack even + when it is advertised by the server. Makes it possible to work + around a limitation in the git for windows implementation together + with the dump git protocol. Defaults to true. diff --git a/send-pack.c b/send-pack.c index 69921f6b65f82a..ba1c6d3b99ba44 100644 --- a/send-pack.c +++ b/send-pack.c @@ -490,7 +490,7 @@ int send_pack(struct repository *r, int need_pack_data = 0; int allow_deleting_refs = 0; int status_report = 0; - int use_sideband = 0; + int use_sideband = 1; int quiet_supported = 0; int agent_supported = 0; int advertise_sid = 0; @@ -514,6 +514,7 @@ int send_pack(struct repository *r, goto out; } + repo_config_get_bool(r, "sendpack.sideband", &use_sideband); repo_config_get_bool(r, "push.negotiate", &push_negotiate); if (push_negotiate) { trace2_region_enter("send_pack", "push_negotiate", r); @@ -538,8 +539,7 @@ int send_pack(struct repository *r, allow_deleting_refs = 1; if (server_supports("ofs-delta")) args->use_ofs_delta = 1; - if (server_supports("side-band-64k")) - use_sideband = 1; + use_sideband = use_sideband && server_supports("side-band-64k"); if (server_supports("quiet")) quiet_supported = 1; if (server_supports("agent")) From 212dcc10ce72083ce0dbfaaa84102a20b558105f Mon Sep 17 00:00:00 2001 From: Bjoern Mueller Date: Wed, 22 Jan 2020 13:49:13 +0100 Subject: [PATCH 038/229] mingw: fix fatal error working on mapped network drives on Windows In 1e64d18 (mingw: do resolve symlinks in `getcwd()`) a problem was introduced that causes git for Windows to stop working with certain mapped network drives (in particular, drives that are mapped to locations with long path names). Error message was "fatal: Unable to read current working directory: No such file or directory". Present change fixes this issue as discussed in https://github.com/git-for-windows/git/issues/2480 Signed-off-by: Bjoern Mueller --- compat/mingw.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..78de2fcb1ee126 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1561,8 +1561,13 @@ char *mingw_getcwd(char *pointer, int len) if (hnd != INVALID_HANDLE_VALUE) { ret = GetFinalPathNameByHandleW(hnd, wpointer, ARRAY_SIZE(wpointer), 0); CloseHandle(hnd); - if (!ret || ret >= ARRAY_SIZE(wpointer)) - return NULL; + if (!ret || ret >= ARRAY_SIZE(wpointer)) { + ret = GetLongPathNameW(cwd, wpointer, ARRAY_SIZE(wpointer)); + if (!ret || ret >= ARRAY_SIZE(wpointer)) { + errno = ret ? ENAMETOOLONG : err_win_to_posix(GetLastError()); + return NULL; + } + } if (xwcstoutf(pointer, normalize_ntpath(wpointer), len) < 0) return NULL; return pointer; From 04b9eee7bed396edb285b70ac86ad162908229f1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Thu, 30 Jan 2020 14:22:27 -0500 Subject: [PATCH 039/229] clink.pl: fix MSVC compile script to handle libcurl-d.lib Update clink.pl to link with either libcurl.lib or libcurl-d.lib depending on whether DEBUG=1 is set. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/vcbuild/scripts/clink.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compat/vcbuild/scripts/clink.pl b/compat/vcbuild/scripts/clink.pl index 3bd824154be381..c4c99d1a11f18c 100755 --- a/compat/vcbuild/scripts/clink.pl +++ b/compat/vcbuild/scripts/clink.pl @@ -56,7 +56,8 @@ # need to use that instead? foreach my $flag (@lflags) { if ($flag =~ /^-LIBPATH:(.*)/) { - foreach my $l ("libcurl_imp.lib", "libcurl.lib") { + my $libcurl = $is_debug ? "libcurl-d.lib" : "libcurl.lib"; + foreach my $l ("libcurl_imp.lib", $libcurl) { if (-f "$1/$l") { $lib = $l; last; From 1387e9c6d9c25ed6951ef2bacfb2264407bfec92 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 31 Jan 2020 11:49:04 +0100 Subject: [PATCH 040/229] mingw: implement a platform-specific `strbuf_realpath()` There is a Win32 API function to resolve symbolic links, and we can use that instead of resolving them manually. Even better, this function also resolves NTFS junction points (which are somewhat similar to bind mounts). This fixes https://github.com/git-for-windows/git/issues/2481. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 76 +++++++++++++++++++++++++++++++++++++++++++ compat/mingw.h | 3 ++ t/t0060-path-utils.sh | 8 +++++ t/t3700-add.sh | 2 +- t/t5601-clone.sh | 7 ++++ 5 files changed, 95 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..b5eb6f04150867 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1545,6 +1545,82 @@ struct tm *localtime_r(const time_t *timep, struct tm *result) } #endif +char *mingw_strbuf_realpath(struct strbuf *resolved, const char *path) +{ + wchar_t wpath[MAX_PATH]; + HANDLE h; + DWORD ret; + int len; + const char *last_component = NULL; + char *append = NULL; + + if (xutftowcs_path(wpath, path) < 0) + return NULL; + + h = CreateFileW(wpath, 0, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, + OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); + + /* + * strbuf_realpath() allows the last path component to not exist. If + * that is the case, now it's time to try without last component. + */ + if (h == INVALID_HANDLE_VALUE && + GetLastError() == ERROR_FILE_NOT_FOUND) { + /* cut last component off of `wpath` */ + wchar_t *p = wpath + wcslen(wpath); + + while (p != wpath) + if (*(--p) == L'/' || *p == L'\\') + break; /* found start of last component */ + + if (p != wpath && (last_component = find_last_dir_sep(path))) { + append = xstrdup(last_component + 1); /* skip directory separator */ + /* + * Do not strip the trailing slash at the drive root, otherwise + * the path would be e.g. `C:` (which resolves to the + * _current_ directory on that drive). + */ + if (p[-1] == L':') + p[1] = L'\0'; + else + *p = L'\0'; + h = CreateFileW(wpath, 0, FILE_SHARE_READ | + FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL); + } + } + + if (h == INVALID_HANDLE_VALUE) { +realpath_failed: + FREE_AND_NULL(append); + return NULL; + } + + ret = GetFinalPathNameByHandleW(h, wpath, ARRAY_SIZE(wpath), 0); + CloseHandle(h); + if (!ret || ret >= ARRAY_SIZE(wpath)) + goto realpath_failed; + + len = wcslen(wpath) * 3; + strbuf_grow(resolved, len); + len = xwcstoutf(resolved->buf, normalize_ntpath(wpath), len); + if (len < 0) + goto realpath_failed; + resolved->len = len; + + if (append) { + /* Use forward-slash, like `normalize_ntpath()` */ + strbuf_complete(resolved, '/'); + strbuf_addstr(resolved, append); + FREE_AND_NULL(append); + } + + return resolved->buf; + +} + char *mingw_getcwd(char *pointer, int len) { wchar_t cwd[MAX_PATH], wpointer[MAX_PATH]; diff --git a/compat/mingw.h b/compat/mingw.h index df02aeb632d8c3..8f3dbed9a7b789 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -39,6 +39,9 @@ static inline void convert_slashes(char *path) #define PATH_SEP ';' char *mingw_query_user_email(void); #define query_user_email mingw_query_user_email +struct strbuf; +char *mingw_strbuf_realpath(struct strbuf *resolved, const char *path); +#define platform_strbuf_realpath mingw_strbuf_realpath /** * Verifies that the specified path is owned by the user running the diff --git a/t/t0060-path-utils.sh b/t/t0060-path-utils.sh index 56faf5fe732ee0..3cdc4738644dbc 100755 --- a/t/t0060-path-utils.sh +++ b/t/t0060-path-utils.sh @@ -281,6 +281,14 @@ test_expect_success SYMLINKS 'real path works on symlinks' ' test_cmp expect actual ' +test_expect_success MINGW 'real path works near drive root' ' + # we need a non-existing path at the drive root; simply skip if C:/xyz exists + if test ! -e C:/xyz + then + test C:/xyz = $(test-tool path-utils real_path C:/xyz) + fi +' + test_expect_success SYMLINKS 'prefix_path works with absolute paths to work tree symlinks' ' ln -s target symlink && echo "symlink" >expect && diff --git a/t/t3700-add.sh b/t/t3700-add.sh index 3647569a4fc53c..d86dc713572d0e 100755 --- a/t/t3700-add.sh +++ b/t/t3700-add.sh @@ -587,7 +587,7 @@ test_expect_success CASE_INSENSITIVE_FS 'path is case-insensitive' ' git add "$downcased" ' -test_expect_failure MINGW 'can add files via NTFS junctions' ' +test_expect_success MINGW 'can add files via NTFS junctions' ' test_when_finished "cmd //c rmdir junction && rm -rf target" && test_create_repo target && cmd //c "mklink /j junction target" && diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 06f6121f82e526..8a9e9352006e98 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -78,6 +78,13 @@ test_expect_success 'clone respects GIT_WORK_TREE' ' ' +test_expect_success CASE_INSENSITIVE_FS 'core.worktree is not added due to path case' ' + + mkdir UPPERCASE && + git clone src "$(pwd)/uppercase" && + test "unset" = "$(git -C UPPERCASE config --default unset core.worktree)" +' + test_expect_success 'clone from hooks' ' test_create_repo r0 && From da5b5effaf3ff87dc191d07b1d3cc4479eeaf980 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 4 Mar 2020 21:55:28 +0100 Subject: [PATCH 041/229] http: use new "best effort" strategy for Secure Channel revoke checking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The native Windows HTTPS backend is based on Secure Channel which lets the caller decide how to handle revocation checking problems caused by missing information in the certificate or offline CRL distribution points. Unfortunately, cURL chose to handle these problems differently than OpenSSL by default: while OpenSSL happily ignores those problems (essentially saying "¯\_(ツ)_/¯"), the Secure Channel backend will error out instead. As a remedy, the "no revoke" mode was introduced, which turns off revocation checking altogether. This is a bit heavy-handed. We support this via the `http.schannelCheckRevoke` setting. In https://github.com/curl/curl/pull/4981, we contributed an opt-in "best effort" strategy that emulates what OpenSSL seems to do. In Git for Windows, we actually want this to be the default. This patch makes it so, introducing it as a new value for the `http.schannelCheckRevoke" setting, which now becmes a tristate: it accepts the values "false", "true" or "best-effort" (defaulting to the last one). Signed-off-by: Johannes Schindelin --- Documentation/config/http.adoc | 12 +++++++----- http.c | 25 +++++++++++++++++++++---- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/Documentation/config/http.adoc b/Documentation/config/http.adoc index b54f62796921c1..76e4550a91e547 100644 --- a/Documentation/config/http.adoc +++ b/Documentation/config/http.adoc @@ -258,11 +258,13 @@ http.sslKeyType:: http.schannelCheckRevoke:: Used to enforce or disable certificate revocation checks in cURL - when http.sslBackend is set to "schannel". Defaults to `true` if - unset. Only necessary to disable this if Git consistently errors - and the message is about checking the revocation status of a - certificate. This option is ignored if cURL lacks support for - setting the relevant SSL option at runtime. + when http.sslBackend is set to "schannel" via "true" and "false", + respectively. Another accepted value is "best-effort" (the default) + in which case revocation checks are performed, but errors due to + revocation list distribution points that are offline are silently + ignored, as well as errors due to certificates missing revocation + list distribution points. This option is ignored if cURL lacks + support for setting the relevant SSL option at runtime. http.schannelUseSSLCAInfo:: As of cURL v7.60.0, the Secure Channel backend can use the diff --git a/http.c b/http.c index 334f33e3233dbb..fdd0bba1fdaa30 100644 --- a/http.c +++ b/http.c @@ -152,7 +152,12 @@ static char *cached_accept_language; static char *http_ssl_backend; -static int http_schannel_check_revoke = 1; +static long http_schannel_check_revoke_mode = +#ifdef CURLSSLOPT_REVOKE_BEST_EFFORT + CURLSSLOPT_REVOKE_BEST_EFFORT; +#else + CURLSSLOPT_NO_REVOKE; +#endif static long http_retry_after = 0; static long http_max_retries = 0; @@ -436,7 +441,19 @@ static int http_options(const char *var, const char *value, } if (!strcmp("http.schannelcheckrevoke", var)) { - http_schannel_check_revoke = git_config_bool(var, value); + if (value && !strcmp(value, "best-effort")) { + http_schannel_check_revoke_mode = +#ifdef CURLSSLOPT_REVOKE_BEST_EFFORT + CURLSSLOPT_REVOKE_BEST_EFFORT; +#else + CURLSSLOPT_NO_REVOKE; + warning(_("%s=%s unsupported by current cURL"), + var, value); +#endif + } else + http_schannel_check_revoke_mode = + (git_config_bool(var, value) ? + 0 : CURLSSLOPT_NO_REVOKE); return 0; } @@ -1175,8 +1192,8 @@ static CURL *get_curl_handle(void) #endif if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) && - !http_schannel_check_revoke) { - curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, (long)CURLSSLOPT_NO_REVOKE); + http_schannel_check_revoke_mode) { + curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, http_schannel_check_revoke_mode); } if (http_proactive_auth != PROACTIVE_AUTH_NONE) From f2d570f5449585e959443cb01e304407d23f1cac Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 9 May 2020 19:24:23 +0200 Subject: [PATCH 042/229] t5505/t5516: fix white-space around redirectors The convention in Git project's shell scripts is to have white-space _before_, but not _after_ the `>` (or `<`). Signed-off-by: Johannes Schindelin --- t/t5505-remote.sh | 6 +++--- t/t5516-fetch-push.sh | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/t/t5505-remote.sh b/t/t5505-remote.sh index d8af3c980008f9..5ea5792e14e32c 100755 --- a/t/t5505-remote.sh +++ b/t/t5505-remote.sh @@ -973,8 +973,8 @@ test_expect_success '"remote show" does not show symbolic refs' ' ( cd three && git remote show origin >output && - ! grep "^ *HEAD$" < output && - ! grep -i stale < output + ! grep "^ *HEAD$" .git/branches/origin && + echo "quux#foom" >.git/branches/origin && git remote rename origin origin && test_path_is_missing .git/branches/origin && test "$(git config remote.origin.url)" = "quux" && diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index fcb039dd6659da..b1e82082b7823f 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -964,7 +964,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'fetch with branches' ' git branch second $the_first_commit && git checkout second && mkdir -p testrepo/.git/branches && - echo ".." > testrepo/.git/branches/branch1 && + echo ".." >testrepo/.git/branches/branch1 && ( cd testrepo && git fetch branch1 && @@ -978,7 +978,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'fetch with branches' ' test_expect_success !WITH_BREAKING_CHANGES 'fetch with branches containing #' ' mk_empty testrepo && mkdir -p testrepo/.git/branches && - echo "..#second" > testrepo/.git/branches/branch2 && + echo "..#second" >testrepo/.git/branches/branch2 && ( cd testrepo && git fetch branch2 && @@ -995,7 +995,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'push with branches' ' test_when_finished "rm -rf .git/branches" && mkdir -p .git/branches && - echo "testrepo" > .git/branches/branch1 && + echo "testrepo" >.git/branches/branch1 && git push branch1 && ( @@ -1011,7 +1011,7 @@ test_expect_success !WITH_BREAKING_CHANGES 'push with branches containing #' ' test_when_finished "rm -rf .git/branches" && mkdir -p .git/branches && - echo "testrepo#branch3" > .git/branches/branch2 && + echo "testrepo#branch3" >.git/branches/branch2 && git push branch2 && ( @@ -1541,7 +1541,7 @@ EOF git init no-thin && git --git-dir=no-thin/.git config receive.unpacklimit 0 && git push no-thin/.git refs/heads/main:refs/heads/foo && - echo modified >> path1 && + echo modified >>path1 && git commit -am modified && git repack -adf && rcvpck="git receive-pack --reject-thin-pack-for-testing" && From 2b91c9d5e76bba082ebd765f2b44e3975bca855f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 12 Sep 2015 12:25:47 +0200 Subject: [PATCH 043/229] t3701: verify that we can add *lots* of files interactively Signed-off-by: Johannes Schindelin --- t/t3701-add-interactive.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/t/t3701-add-interactive.sh b/t/t3701-add-interactive.sh index 6e120a40011238..cb09158c214768 100755 --- a/t/t3701-add-interactive.sh +++ b/t/t3701-add-interactive.sh @@ -1204,6 +1204,27 @@ test_expect_success 'checkout -p patch editing of added file' ' ) ' +test_expect_success EXPENSIVE 'add -i with a lot of files' ' + git reset --hard && + x160=0123456789012345678901234567890123456789 && + x160=$x160$x160$x160$x160 && + y= && + i=0 && + while test $i -le 200 + do + name=$(printf "%s%03d" $x160 $i) && + echo $name >$name && + git add -N $name && + y="${y}y$LF" && + i=$(($i+1)) || + exit 1 + done && + echo "$y" | git add -p -- . && + git diff --cached >staged && + test_line_count = 1407 staged && + git reset --hard +' + test_expect_success 'show help from add--helper' ' git reset --hard && cat >expect <<-EOF && From 1e840725659f4d4405fb6b2bdf45075f39a868fb Mon Sep 17 00:00:00 2001 From: Luke Bonanomi Date: Wed, 24 Jun 2020 07:45:52 -0400 Subject: [PATCH 044/229] commit: accept "scissors" with CR/LF line endings This change enhances `git commit --cleanup=scissors` by detecting scissors lines ending in either LF (UNIX-style) or CR/LF (DOS-style). Regression tests are included to specifically test for trailing comments after a CR/LF-terminated scissors line. Signed-off-by: Luke Bonanomi Signed-off-by: Johannes Schindelin --- t/t7502-commit-porcelain.sh | 42 +++++++++++++++++++++++++++++++++++++ wt-status.c | 13 +++++++++--- 2 files changed, 52 insertions(+), 3 deletions(-) diff --git a/t/t7502-commit-porcelain.sh b/t/t7502-commit-porcelain.sh index 2adfe70b3d1c25..dbacc73e9d3537 100755 --- a/t/t7502-commit-porcelain.sh +++ b/t/t7502-commit-porcelain.sh @@ -600,6 +600,48 @@ test_expect_success 'cleanup commit messages (scissors option,-F,-e, scissors on test_must_be_empty actual ' +test_expect_success 'helper-editor' ' + + write_script lf-to-crlf.sh <<-\EOF + sed "s/\$/Q/" <"$1" | tr Q "\\015" >"$1".new && + mv -f "$1".new "$1" + EOF +' + +test_expect_success 'cleanup commit messages (scissors option,-F,-e, CR/LF line endings)' ' + + test_config core.editor "\"$PWD/lf-to-crlf.sh\"" && + scissors="# ------------------------ >8 ------------------------" && + + test_write_lines >text \ + "# Keep this comment" "" " $scissors" \ + "# Keep this comment, too" "$scissors" \ + "# Remove this comment" "$scissors" \ + "Remove this comment, too" && + + test_write_lines >expect \ + "# Keep this comment" "" " $scissors" \ + "# Keep this comment, too" && + + git commit --cleanup=scissors -e -F text --allow-empty && + git cat-file -p HEAD >raw && + sed -e "1,/^\$/d" raw >actual && + test_cmp expect actual +' + +test_expect_success 'cleanup commit messages (scissors option,-F,-e, scissors on first line, CR/LF line endings)' ' + + scissors="# ------------------------ >8 ------------------------" && + test_write_lines >text \ + "$scissors" \ + "# Remove this comment and any following lines" && + cp text /tmp/test2-text && + git commit --cleanup=scissors -e -F text --allow-empty --allow-empty-message && + git cat-file -p HEAD >raw && + sed -e "1,/^\$/d" raw >actual && + test_must_be_empty actual +' + test_expect_success 'cleanup commit messages (strip option,-F)' ' echo >>negative && diff --git a/wt-status.c b/wt-status.c index f7b0dc29d51e1b..18950ab92d96b2 100644 --- a/wt-status.c +++ b/wt-status.c @@ -40,7 +40,7 @@ #define UF_DELAY_WARNING_IN_MS (2 * 1000) static const char cut_line[] = -"------------------------ >8 ------------------------\n"; +"------------------------ >8 ------------------------"; static char default_wt_status_colors[][COLOR_MAXLEN] = { GIT_COLOR_NORMAL, /* WT_STATUS_HEADER */ @@ -1123,15 +1123,22 @@ static void wt_longstatus_print_other(struct wt_status *s, status_printf_ln(s, GIT_COLOR_NORMAL, "%s", ""); } +static inline int starts_with_newline(const char *p) +{ + return *p == '\n' || (*p == '\r' && p[1] == '\n'); +} + size_t wt_status_locate_end(const char *s, size_t len) { const char *p; struct strbuf pattern = STRBUF_INIT; strbuf_addf(&pattern, "\n%s %s", comment_line_str, cut_line); - if (starts_with(s, pattern.buf + 1)) + if (starts_with(s, pattern.buf + 1) && + starts_with_newline(s + pattern.len - 1)) len = 0; - else if ((p = strstr(s, pattern.buf))) { + else if ((p = strstr(s, pattern.buf)) && + starts_with_newline(p + pattern.len)) { size_t newlen = p - s + 1; if (newlen < len) len = newlen; From 0f0c72b9125b3141ff5550518dc25ccd2290ec6a Mon Sep 17 00:00:00 2001 From: Jens Glathe Date: Tue, 2 Jun 2020 12:12:25 +0200 Subject: [PATCH 045/229] t0014: fix indentation For some reason, this test case was indented with 4 spaces instead of 1 horizontal tab. The other test cases in the same test script are fine. Signed-off-by: Jens Glathe Signed-off-by: Johannes Schindelin --- t/t0014-alias.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/t/t0014-alias.sh b/t/t0014-alias.sh index cbc447b4814e78..d2501fc8d643b7 100755 --- a/t/t0014-alias.sh +++ b/t/t0014-alias.sh @@ -60,10 +60,10 @@ test_expect_success HAVE_DEPRECATED 'looping aliases - deprecated builtins' ' #' test_expect_success 'run-command formats empty args properly' ' - test_must_fail env GIT_TRACE=1 git frotz a "" b " " c 2>actual.raw && - sed -ne "/run_command:/s/.*trace: run_command: //p" actual.raw >actual && - echo "git-frotz a '\'''\'' b '\'' '\'' c" >expect && - test_cmp expect actual + test_must_fail env GIT_TRACE=1 git frotz a "" b " " c 2>actual.raw && + sed -ne "/run_command:/s/.*trace: run_command: //p" actual.raw >actual && + echo "git-frotz a '\'''\'' b '\'' '\'' c" >expect && + test_cmp expect actual ' test_expect_success 'tracing a shell alias with arguments shows trace of prepared command' ' From 6ab79b840072db71d54a98233626a7f749134856 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 12 Aug 2020 15:06:17 +0000 Subject: [PATCH 046/229] git-gui: accommodate for intent-to-add files As of Git v2.28.0, the diff for files staged via `git add -N` marks them as new files. Git GUI was ill-prepared for that, and this patch teaches Git GUI about them. Please note that this will not even fix things with v2.28.0, as the `rp/apply-cached-with-i-t-a` patches are required on Git's side, too. This fixes https://github.com/git-for-windows/git/issues/2779 Signed-off-by: Johannes Schindelin Signed-off-by: Pratyush Yadav --- git-gui/git-gui.sh | 2 ++ git-gui/lib/diff.tcl | 12 ++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/git-gui/git-gui.sh b/git-gui/git-gui.sh index 15dd2b3a84ccd1..b8737c10bce2ae 100755 --- a/git-gui/git-gui.sh +++ b/git-gui/git-gui.sh @@ -1991,6 +1991,7 @@ set all_icons(U$ui_index) file_merge set all_icons(T$ui_index) file_statechange set all_icons(_$ui_workdir) file_plain +set all_icons(A$ui_workdir) file_plain set all_icons(M$ui_workdir) file_mod set all_icons(D$ui_workdir) file_question set all_icons(U$ui_workdir) file_merge @@ -2017,6 +2018,7 @@ foreach i { {A_ {mc "Staged for commit"}} {AM {mc "Portions staged for commit"}} {AD {mc "Staged for commit, missing"}} + {AA {mc "Intended to be added"}} {_D {mc "Missing"}} {D_ {mc "Staged for removal"}} diff --git a/git-gui/lib/diff.tcl b/git-gui/lib/diff.tcl index 8be1a613fbe01f..d25a9bbdc4abde 100644 --- a/git-gui/lib/diff.tcl +++ b/git-gui/lib/diff.tcl @@ -556,7 +556,8 @@ proc apply_or_revert_hunk {x y revert} { if {$current_diff_side eq $ui_index} { set failed_msg [mc "Failed to unstage selected hunk."] lappend apply_cmd --reverse --cached - if {[string index $mi 0] ne {M}} { + set file_state [string index $mi 0] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } @@ -569,7 +570,8 @@ proc apply_or_revert_hunk {x y revert} { lappend apply_cmd --cached } - if {[string index $mi 1] ne {M}} { + set file_state [string index $mi 1] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } @@ -661,7 +663,8 @@ proc apply_or_revert_range_or_line {x y revert} { set failed_msg [mc "Failed to unstage selected line."] set to_context {+} lappend apply_cmd --reverse --cached - if {[string index $mi 0] ne {M}} { + set file_state [string index $mi 0] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } @@ -676,7 +679,8 @@ proc apply_or_revert_range_or_line {x y revert} { lappend apply_cmd --cached } - if {[string index $mi 1] ne {M}} { + set file_state [string index $mi 1] + if {$file_state ne {M} && $file_state ne {A}} { unlock_index return } From 5ecb415357e7a6843e62d3a9bb61bb770893b7fb Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Sun, 6 Oct 2019 18:43:57 +0100 Subject: [PATCH 047/229] vcpkg_install: add comment regarding slow network connections The vcpkg downloads may not succeed. Warn careful readers of the time out. A simple retry will usually resolve the issue. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- compat/vcbuild/vcpkg_install.bat | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index bcbbf536af3141..8330d8120fb511 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -80,6 +80,12 @@ REM ================================================================ :sub__install_one echo Installing package %1... + REM vcpkg may not be reliable on slow, intermittent or proxy + REM connections, see e.g. + REM https://social.msdn.microsoft.com/Forums/windowsdesktop/en-US/4a8f7be5-5e15-4213-a7bb-ddf424a954e6/winhttpsendrequest-ends-with-12002-errorhttptimeout-after-21-seconds-no-matter-what-timeout?forum=windowssdk + REM which explains the hidden 21 second timeout + REM (last post by Dave : Microsoft - Windows Networking team) + .\vcpkg.exe install %1:%arch% IF ERRORLEVEL 1 ( EXIT /B 1 ) From 7550292ac80de4fe8848aef68600253e349f443b Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 6 Dec 2020 18:39:26 +0100 Subject: [PATCH 048/229] vcbuild: stop hard-coding OpenSSL as a dependency Git for Windows' default SSL backend is actually Secure Channel. Let's not hard-code any backend, just ask for _any_ SSL backend. This is necessary because we cannot ask for `schannel`, as https://github.com/microsoft/vcpkg/pull/46459 removed the option to specify that as a feature. Signed-off-by: Dennis Ameling Signed-off-by: Johannes Schindelin --- compat/vcbuild/vcpkg_install.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/vcbuild/vcpkg_install.bat b/compat/vcbuild/vcpkg_install.bat index 8da212487ae97d..b6600687dc581a 100644 --- a/compat/vcbuild/vcpkg_install.bat +++ b/compat/vcbuild/vcpkg_install.bat @@ -127,5 +127,5 @@ set features= goto :EOF :curl_features -set features=[core,openssl] +set features=[core,ssl] goto :EOF From dccd3fda20068c53a452f7cbb5e26aac0c354f87 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Mon, 19 Jul 2021 13:02:16 +0200 Subject: [PATCH 049/229] cmake(): allow setting HOST_CPU for cross-compilation Git's regular Makefile mentions that HOST_CPU should be defined when cross-compiling Git: https://github.com/git-for-windows/git/blob/37796bca76ef4180c39ee508ca3e42c0777ba444/Makefile#L438-L439 This is then used to set the GIT_HOST_CPU variable when compiling Git: https://github.com/git-for-windows/git/blob/37796bca76ef4180c39ee508ca3e42c0777ba444/Makefile#L1337-L1341 Then, when the user runs `git version --build-options`, it returns that value: https://github.com/git-for-windows/git/blob/37796bca76ef4180c39ee508ca3e42c0777ba444/help.c#L658 This commit adds the same functionality to the CMake configuration. Users can now set -DHOST_CPU= to set the target architecture. Signed-off-by: Dennis Ameling --- .github/workflows/main.yml | 2 +- contrib/buildsystems/CMakeLists.txt | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 94324e4829ea5c..81619fd3abfbf2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -223,7 +223,7 @@ jobs: shell: bash run: | cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/${{ matrix.arch }}-windows \ - -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=${{ matrix.arch }} -DVCPKG_ARCH=${{ matrix.arch }}-windows + -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON -DCMAKE_GENERATOR_PLATFORM=${{ matrix.arch }} -DVCPKG_ARCH=${{ matrix.arch }}-windows -DHOST_CPU=${{ matrix.arch }} - name: MSBuild run: | $sln = if (Test-Path git.slnx) { 'git.slnx' } else { 'git.sln' } diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 77afcdfd09c061..5ce6fc48cc61ee 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -212,7 +212,14 @@ endif() #default behaviour include_directories(${CMAKE_SOURCE_DIR}) -add_compile_definitions(GIT_HOST_CPU="${CMAKE_SYSTEM_PROCESSOR}") + +# When cross-compiling, define HOST_CPU as the canonical name of the CPU on +# which the built Git will run (for instance "x86_64"). +if(NOT HOST_CPU) + add_compile_definitions(GIT_HOST_CPU="${CMAKE_SYSTEM_PROCESSOR}") +else() + add_compile_definitions(GIT_HOST_CPU="${HOST_CPU}") +endif() add_compile_definitions(SHA256_BLK INTERNAL_QSORT RUNTIME_PREFIX) add_compile_definitions(NO_OPENSSL SHA1_DC SHA1DC_NO_STANDARD_INCLUDES SHA1DC_INIT_SAFE_HASH_DEFAULT=0 From a5d8287220228b67ed9040ad5684366c75493840 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 2 Apr 2021 22:50:54 +0200 Subject: [PATCH 050/229] mingw: allow for longer paths in `parse_interpreter()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As reported in https://github.com/newren/git-filter-repo/pull/225, it looks like 99 bytes is not really sufficient to represent e.g. the full path to Python when installed via Windows Store (and this path is used in the hasb bang line when installing scripts via `pip`). Let's increase it to what is probably the maximum sensible path size: MAX_PATH. This makes `parse_interpreter()` in line with what `lookup_prog()` handles. Signed-off-by: Johannes Schindelin Signed-off-by: Vilius Šumskas --- compat/mingw.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..31ba74a7f113a7 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1673,7 +1673,7 @@ static const char *quote_arg_msys2(const char *arg) static const char *parse_interpreter(const char *cmd) { - static char buf[100]; + static char buf[MAX_PATH]; char *p, *opt; ssize_t n; /* read() can return negative values */ int fd; From c2c309ca6451c790e669fc35cac1b06a4a31dcb1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 17 May 2021 10:46:52 +0200 Subject: [PATCH 051/229] compat/vcbuild: document preferred way to build in Visual Studio We used to have that `make vcxproj` hack, but a hack it is. In the meantime, we have a much cleaner solution: using CMake, either explicitly, or even more conveniently via Visual Studio's built-in CMake support (simply open Git's top-level directory via File>Open>Folder...). Let's let the `README` reflect this. Signed-off-by: Johannes Schindelin --- compat/vcbuild/README | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/compat/vcbuild/README b/compat/vcbuild/README index 29ec1d0f104b80..5c71ea2daa4017 100644 --- a/compat/vcbuild/README +++ b/compat/vcbuild/README @@ -37,27 +37,17 @@ The Steps to Build Git with VS2015 or VS2017 from the command line. ================================================================ -Alternatively, run `make vcxproj` and then load the generated `git.sln` in -Visual Studio. The initial build will install the vcpkg system and build the +Alternatively, just open Git's top-level directory in Visual Studio, via +`File>Open>Folder...`. This will use CMake internally to generate the +project definitions. It will also install the vcpkg system and build the dependencies automatically. This will take a while. -Instead of generating the `git.sln` file yourself (which requires a full Git -for Windows SDK), you may want to consider fetching the `vs/master` branch of -https://github.com/git-for-windows/git instead (which is updated automatically -via CI running `make vcxproj`). The `vs/master` branch does not require a Git -for Windows to build, but you can run the test scripts in a regular Git Bash. - -Note that `make vcxproj` will automatically add and commit the generated `.sln` -and `.vcxproj` files to the repo. This is necessary to allow building a -fully-testable Git in Visual Studio, where a regular Git Bash can be used to -run the test scripts (as opposed to a full Git for Windows SDK): a number of -build targets, such as Git commands implemented as Unix shell scripts (where -`@@SHELL_PATH@@` and other placeholders are interpolated) require a full-blown -Git for Windows SDK (which is about 10x the size of a regular Git for Windows -installation). - -If your plan is to open a Pull Request with Git for Windows, it is a good idea -to drop this commit before submitting. +You can also generate the Visual Studio solution manually by downloading +and running CMake explicitly rather than letting Visual Studio doing +that implicitly. + +Another, deprecated option is to run `make vcxproj`. This option is +superseded by the CMake-based build, and will be removed at some point. ================================================================ The Steps of Build Git with VS2008 From faf4756130169e8bc912e67caf6e73cdf734969c Mon Sep 17 00:00:00 2001 From: Pascal Muller Date: Wed, 23 Jun 2021 21:21:10 +0200 Subject: [PATCH 052/229] http: optionally send SSL client certificate This adds support for a new http.sslAutoClientCert config value. In cURL 7.77 or later the schannel backend does not automatically send client certificates from the Windows Certificate Store anymore. This config value is only used if http.sslBackend is set to "schannel", and can be used to opt in to the old behavior and force cURL to send client certificates. This fixes https://github.com/git-for-windows/git/issues/3292 Signed-off-by: Pascal Muller --- Documentation/config/http.adoc | 5 +++++ git-curl-compat.h | 8 ++++++++ http.c | 24 +++++++++++++++++++++--- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/Documentation/config/http.adoc b/Documentation/config/http.adoc index 76e4550a91e547..acfdd5afd0a5e9 100644 --- a/Documentation/config/http.adoc +++ b/Documentation/config/http.adoc @@ -274,6 +274,11 @@ http.schannelUseSSLCAInfo:: when the `schannel` backend was configured via `http.sslBackend`, unless `http.schannelUseSSLCAInfo` overrides this behavior. +http.sslAutoClientCert:: + As of cURL v7.77.0, the Secure Channel backend won't automatically + send client certificates from the Windows Certificate Store anymore. + To opt in to the old behavior, http.sslAutoClientCert can be set. + http.pinnedPubkey:: Public key of the https service. It may either be the filename of a PEM or DER encoded public key file or a string starting with diff --git a/git-curl-compat.h b/git-curl-compat.h index 25678b5dbd2c80..c4bf7e6b04ebb3 100644 --- a/git-curl-compat.h +++ b/git-curl-compat.h @@ -52,6 +52,14 @@ #define GIT_CURL_HAVE_CURLINFO_RETRY_AFTER 1 #endif +/** + * CURLSSLOPT_AUTO_CLIENT_CERT was added in 7.77.0, released in May + * 2021. + */ +#if LIBCURL_VERSION_NUM >= 0x074d00 +#define GIT_CURL_HAVE_CURLSSLOPT_AUTO_CLIENT_CERT +#endif + /** * CURLOPT_PROTOCOLS_STR and CURLOPT_REDIR_PROTOCOLS_STR were added in 7.85.0, * released in August 2022. diff --git a/http.c b/http.c index fdd0bba1fdaa30..f3eaf4c6b49c77 100644 --- a/http.c +++ b/http.c @@ -170,6 +170,8 @@ static long http_max_retry_time = 300; */ static int http_schannel_use_ssl_cainfo; +static int http_auto_client_cert; + static int always_auth_proactively(void) { return http_proactive_auth != PROACTIVE_AUTH_NONE && @@ -462,6 +464,11 @@ static int http_options(const char *var, const char *value, return 0; } + if (!strcmp("http.sslautoclientcert", var)) { + http_auto_client_cert = git_config_bool(var, value); + return 0; + } + if (!strcmp("http.minsessions", var)) { min_curl_sessions = git_config_int(var, value, ctx->kvi); if (min_curl_sessions > 1) @@ -1191,9 +1198,20 @@ static CURL *get_curl_handle(void) } #endif - if (http_ssl_backend && !strcmp("schannel", http_ssl_backend) && - http_schannel_check_revoke_mode) { - curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, http_schannel_check_revoke_mode); + if (http_ssl_backend && !strcmp("schannel", http_ssl_backend)) { + long ssl_options = 0; + if (http_schannel_check_revoke_mode) { + ssl_options |= http_schannel_check_revoke_mode; + } + + if (http_auto_client_cert) { +#ifdef GIT_CURL_HAVE_CURLSSLOPT_AUTO_CLIENT_CERT + ssl_options |= CURLSSLOPT_AUTO_CLIENT_CERT; +#endif + } + + if (ssl_options) + curl_easy_setopt(result, CURLOPT_SSL_OPTIONS, ssl_options); } if (http_proactive_auth != PROACTIVE_AUTH_NONE) From 3baa647fe37c9b5827f7710ccd7e649967354eed Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Thu, 5 Aug 2021 19:11:59 -0400 Subject: [PATCH 053/229] ci: run `contrib/subtree` tests in CI builds Because `git subtree` (unlike most other `contrib` modules) is included as part of the standard release of Git for Windows, its stability should be verified as consistently as it is for the rest of git. By including the `git subtree` tests in the CI workflow, these tests are as much of a gate to merging and indicator of stability as the standard test suite. Signed-off-by: Victoria Dye --- ci/run-build-and-tests.sh | 4 ++++ ci/run-test-slice.sh | 3 +++ 2 files changed, 7 insertions(+) diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 6a3b43b366048d..e6ab3b7e04e808 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -72,5 +72,9 @@ case "$jobname" in ;; esac +case " $MAKE_TARGETS " in +*" all "*) make -C contrib/subtree test;; +esac + check_unignored_build_artifacts save_good_tree diff --git a/ci/run-test-slice.sh b/ci/run-test-slice.sh index d0063efc424a58..9c9a7169fb014f 100755 --- a/ci/run-test-slice.sh +++ b/ci/run-test-slice.sh @@ -15,4 +15,7 @@ if [ "$1" == "1" ] ; then group "Run unit tests" make --quiet -C t unit-tests-test-tool fi +# Run the git subtree tests only if main tests succeeded +test 0 != "$1" || make -C contrib/subtree test + check_unignored_build_artifacts From d6d4c1df0bdcd2f2bc5ee3b1c3dc986b0c006f22 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Mon, 10 May 2021 16:47:40 +0100 Subject: [PATCH 054/229] CMake: show Win32 and Generator_platform build-option values Ensure key CMake option values are part of the CMake output to facilitate user support when tool updates impact the wider CMake actions, particularly ongoing 'improvements' in Visual Studio. These CMake displays perform the same function as the build-options.txt provided in the main Git for Windows. CMake is already chatty. The setting of CMAKE_EXPORT_COMPILE_COMMANDS is also reported. Include the environment's CMAKE_EXPORT_COMPILE_COMMANDS value which may have been propogated to CMake's internal value. Testing the CMAKE_EXPORT_COMPILE_COMMANDS processing can be difficult in the Visual Studio environment, as it may be cached in many places. The 'environment' may include the OS, the user shell, CMake's own environment, along with the Visual Studio presets and caches. See previous commit for arefacts that need removing for a clean test. Signed-off-by: Philip Oakley --- contrib/buildsystems/CMakeLists.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index a68f44d5f8f88a..9c9104bfaa3ba6 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -63,10 +63,20 @@ endif() if(NOT DEFINED CMAKE_EXPORT_COMPILE_COMMANDS) set(CMAKE_EXPORT_COMPILE_COMMANDS TRUE) + message("settting CMAKE_EXPORT_COMPILE_COMMANDS: ${CMAKE_EXPORT_COMPILE_COMMANDS}") endif() if(USE_VCPKG) set(VCPKG_DIR "${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg") + message("WIN32: ${WIN32}") # show its underlying text values + message("VCPKG_DIR: ${VCPKG_DIR}") + message("VCPKG_ARCH: ${VCPKG_ARCH}") # maybe unset + message("MSVC: ${MSVC}") + message("CMAKE_GENERATOR: ${CMAKE_GENERATOR}") + message("CMAKE_CXX_COMPILER_ID: ${CMAKE_CXX_COMPILER_ID}") + message("CMAKE_GENERATOR_PLATFORM: ${CMAKE_GENERATOR_PLATFORM}") + message("CMAKE_EXPORT_COMPILE_COMMANDS: ${CMAKE_EXPORT_COMPILE_COMMANDS}") + message("ENV(CMAKE_EXPORT_COMPILE_COMMANDS): $ENV{CMAKE_EXPORT_COMPILE_COMMANDS}") if(NOT EXISTS ${VCPKG_DIR}) message("Initializing vcpkg and building the Git's dependencies (this will take a while...)") execute_process(COMMAND ${CMAKE_SOURCE_DIR}/compat/vcbuild/vcpkg_install.bat ${VCPKG_ARCH}) From c4eeceb1fc78f935b1f721f0c884580c9032268c Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 13 Apr 2022 14:54:43 -0400 Subject: [PATCH 055/229] compat/mingw.c: do not warn when failing to get owner In the case of Git for Windows (say, in a Git Bash window) running in a Windows Subsystem for Linux (WSL) directory, the GetNamedSecurityInfoW() call in is_path_owned_By_current_side() returns an error code other than ERROR_SUCCESS. This is consistent behavior across this boundary. In these cases, the owner would always be different because the WSL owner is a different entity than the Windows user. The change here is to suppress the error message that looks like this: error: failed to get owner for '//wsl.localhost/...' (1) Before this change, this warning happens for every Git command, regardless of whether the directory is marked with safe.directory. Signed-off-by: Derrick Stolee --- compat/mingw.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..2675f4dc7e4df4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3408,9 +3408,7 @@ int is_path_owned_by_current_sid(const char *path, struct strbuf *report) DACL_SECURITY_INFORMATION, &sid, NULL, NULL, NULL, &descriptor); - if (err != ERROR_SUCCESS) - error(_("failed to get owner for '%s' (%ld)"), path, err); - else if (sid && IsValidSid(sid)) { + if (err == ERROR_SUCCESS && sid && IsValidSid(sid)) { /* Now, verify that the SID matches the current user's */ static PSID current_user_sid; static HANDLE linked_token; From 294709d8c713f1afb50957325eeeb138b6ff9062 Mon Sep 17 00:00:00 2001 From: Rafael Kitover Date: Tue, 12 Apr 2022 19:53:33 +0000 Subject: [PATCH 056/229] mingw: $env:TERM="xterm-256color" for newer OSes For Windows builds >= 15063 set $env:TERM to "xterm-256color" instead of "cygwin" because they have a more capable console system that supports this. Also set $env:COLORTERM="truecolor" if unset. $env:TERM is initialized so that ANSI colors in color.c work, see 29a3963484 (Win32: patch Windows environment on startup, 2012-01-15). See git-for-windows/git#3629 regarding problems caused by always setting $env:TERM="cygwin". This is the same heuristic used by the Cygwin runtime. Signed-off-by: Rafael Kitover Signed-off-by: Johannes Schindelin --- compat/mingw.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..02a7f6faaa0bfc 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3220,9 +3220,20 @@ static void setup_windows_environment(void) convert_slashes(tmp); } - /* simulate TERM to enable auto-color (see color.c) */ - if (!getenv("TERM")) - setenv("TERM", "cygwin", 1); + + /* + * Make sure TERM is set up correctly to enable auto-color + * (see color.c .) Use "cygwin" for older OS releases which + * works correctly with MSYS2 utilities on older consoles. + */ + if (!getenv("TERM")) { + if ((GetVersion() >> 16) < 15063) + setenv("TERM", "cygwin", 0); + else { + setenv("TERM", "xterm-256color", 0); + setenv("COLORTERM", "truecolor", 0); + } + } /* calculate HOME if not set */ if (!getenv("HOME")) { From a2a41afdc334d7727a65d8ddcdd2ad1375d81a10 Mon Sep 17 00:00:00 2001 From: Christopher Degawa Date: Sat, 28 May 2022 14:53:54 -0500 Subject: [PATCH 057/229] winansi: check result and Buffer before using Name NtQueryObject under Wine can return a success but fill out no name. In those situations, Wine will set Buffer to NULL, and set result to the sizeof(OBJECT_NAME_INFORMATION). Running a command such as echo "$(git.exe --version 2>/dev/null)" will crash due to a NULL pointer dereference when the code attempts to null terminate the buffer, although, weirdly, removing the subshell or redirecting stdout to a file will not trigger the crash. Code has been added to also check Buffer and Length to ensure the check is as robust as possible due to the current behavior being fragile at best, and could potentially change in the future This code is based on the behavior of NtQueryObject under wine and reactos. Signed-off-by: Christopher Degawa --- compat/winansi.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compat/winansi.c b/compat/winansi.c index 088734a1dfa8f7..599f978b26cbed 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -537,6 +537,9 @@ static void detect_msys_tty(int fd) if (!NT_SUCCESS(NtQueryObject(h, ObjectNameInformation, buffer, sizeof(buffer) - 2, &result))) return; + if (result < sizeof(*nameinfo) || !nameinfo->Name.Buffer || + !nameinfo->Name.Length) + return; name = nameinfo->Name.Buffer; name[nameinfo->Name.Length / sizeof(*name)] = 0; From 0ce3714d2fd2cdf3dd7464deb04b6c239b7fc5ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E5=8D=93=E8=AF=86?= Date: Sun, 16 Jan 2022 03:38:33 +0800 Subject: [PATCH 058/229] Add config option `windows.appendAtomically` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Atomic append on windows is only supported on local disk files, and it may cause errors in other situations, e.g. network file system. If that is the case, this config option should be used to turn atomic append off. Co-Authored-By: Johannes Schindelin Signed-off-by: 孙卓识 Signed-off-by: Johannes Schindelin --- Documentation/config.adoc | 2 ++ Documentation/config/windows.adoc | 4 ++++ compat/mingw.c | 36 ++++++++++++++++++++++++++++--- 3 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 Documentation/config/windows.adoc diff --git a/Documentation/config.adoc b/Documentation/config.adoc index ceb13ade6e4fa2..af689c9bb87555 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -620,4 +620,6 @@ include::config/versionsort.adoc[] include::config/web.adoc[] +include::config/windows.adoc[] + include::config/worktree.adoc[] diff --git a/Documentation/config/windows.adoc b/Documentation/config/windows.adoc new file mode 100644 index 00000000000000..fdaaf1c65504f3 --- /dev/null +++ b/Documentation/config/windows.adoc @@ -0,0 +1,4 @@ +windows.appendAtomically:: + By default, append atomic API is used on windows. But it works only with + local disk files, if you're working on a network file system, you should + set it false to turn it off. diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..3126ad081a39b1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -9,6 +9,7 @@ #include "environment.h" #include "repository.h" #include "gettext.h" +#include "repository.h" #include "run-command.h" #include "strbuf.h" #include "symlinks.h" @@ -834,6 +835,7 @@ static int is_local_named_pipe_path(const char *filename) int mingw_open (const char *filename, int oflags, ...) { + static int append_atomically = -1; typedef int (*open_fn_t)(wchar_t const *wfilename, int oflags, ...); va_list args; unsigned mode; @@ -853,7 +855,16 @@ int mingw_open (const char *filename, int oflags, ...) return -1; } - if ((oflags & O_APPEND) && !is_local_named_pipe_path(filename)) + /* + * Only set append_atomically to default value(1) when repo is initialized + * and fail to get config value + */ + if ((oflags & O_APPEND) && append_atomically < 0 && the_repository && the_repository->commondir && + repo_config_get_bool(the_repository, "windows.appendatomically", &append_atomically)) + append_atomically = 1; + + if (append_atomically && (oflags & O_APPEND) && + !is_local_named_pipe_path(filename)) open_fn = mingw_open_append; else if (!(oflags & ~(O_ACCMODE | O_NOINHERIT))) open_fn = mingw_open_existing; @@ -1032,9 +1043,28 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) /* check if fd is a pipe */ HANDLE h = (HANDLE) _get_osfhandle(fd); - if (GetFileType(h) != FILE_TYPE_PIPE) + if (GetFileType(h) != FILE_TYPE_PIPE) { + if (orig == EINVAL) { + wchar_t path[MAX_PATH]; + DWORD ret = GetFinalPathNameByHandleW(h, path, + ARRAY_SIZE(path), 0); + UINT drive_type = ret > 0 && ret < ARRAY_SIZE(path) ? + GetDriveTypeW(path) : DRIVE_UNKNOWN; + + /* + * The default atomic append causes such an error on + * network file systems, in such a case, it should be + * turned off via config. + * + * `drive_type` of UNC path: DRIVE_NO_ROOT_DIR + */ + if (DRIVE_NO_ROOT_DIR == drive_type || DRIVE_REMOTE == drive_type) + warning("invalid write operation detected; you may try:\n" + "\n\tgit config windows.appendAtomically false"); + } + errno = orig; - else if (orig == EINVAL) + } else if (orig == EINVAL) errno = EPIPE; else { DWORD buf_size; From 145420e8a2e0f4f0346ce972a3f34aad185cc952 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 4 Sep 2017 11:59:45 +0200 Subject: [PATCH 059/229] mingw: change core.fsyncObjectFiles = 1 by default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit From the documentation of said setting: This boolean will enable fsync() when writing object files. This is a total waste of time and effort on a filesystem that orders data writes properly, but can be useful for filesystems that do not use journalling (traditional UNIX filesystems) or that only journal metadata and not file contents (OS X’s HFS+, or Linux ext3 with "data=writeback"). The most common file system on Windows (NTFS) does not guarantee that order, therefore a sudden loss of power (or any other event causing an unclean shutdown) would cause corrupt files (i.e. files filled with NULs). Therefore we need to change the default. Note that the documentation makes it sound as if this causes really bad performance. In reality, writing loose objects is something that is done only rarely, and only a handful of files at a time. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 3126ad081a39b1..6c072900415c4a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -18,6 +18,7 @@ #include "win32/exit-process.h" #include "win32/lazyload.h" #include "wrapper.h" +#include "write-or-die.h" #include #include #include @@ -3799,6 +3800,7 @@ int wmain(int argc, const wchar_t **wargv) maybe_redirect_std_handles(); adjust_symlink_flags(); + fsync_object_files = 1; /* determine size of argv and environ conversion buffer */ maxlen = wcslen(wargv[0]); From 7214054e3e03eed7cb68d8d2b1adb76a28482396 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20A=C3=9Fhauer?= Date: Sun, 10 Jul 2022 11:27:25 +0200 Subject: [PATCH 060/229] MinGW: link as terminal server aware MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Whith Windows 2000, Microsoft introduced a flag to the PE header to mark executables as "terminal server aware". Windows terminal servers provide a redirected Windows directory and redirected registry hives when launching legacy applications without this flag set. Since we do not use any INI files in the Windows directory and don't write to the registry, we don't need this additional preparation. Telling the OS that we don't need this should provide slightly improved startup times in terminal server environments. When building for supported Windows Versions with MSVC the /TSAWARE linker flag is automatically set, but MinGW requires us to set the --tsaware flag manually. This partially addresses https://github.com/git-for-windows/git/issues/3935. Signed-off-by: Matthias Aßhauer --- config.mak.uname | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.mak.uname b/config.mak.uname index 820c5c0e49084a..9a9e5065aba310 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -733,7 +733,7 @@ ifeq ($(uname_S),MINGW) DEFAULT_HELP_FORMAT = html HAVE_PLATFORM_PROCINFO = YesPlease CSPRNG_METHOD = rtlgenrandom - BASIC_LDFLAGS += -municode + BASIC_LDFLAGS += -municode -Wl,--tsaware COMPAT_CFLAGS += -DNOGDI -Icompat -Icompat/win32 COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ From 3bea54e5429bc740b4cedb2898bb8019a99a4fc8 Mon Sep 17 00:00:00 2001 From: Kiel Hurley Date: Wed, 2 Nov 2022 22:56:16 +1300 Subject: [PATCH 061/229] Fix Windows version resources Add FileVersion, which is a required field As not all required fields were present, none were being included Fixes #4090 Signed-off-by: Kiel Hurley --- git.rc.in | 1 + 1 file changed, 1 insertion(+) diff --git a/git.rc.in b/git.rc.in index e69444eef3f0c5..460ea39561b87f 100644 --- a/git.rc.in +++ b/git.rc.in @@ -12,6 +12,7 @@ BEGIN VALUE "OriginalFilename", "git.exe\0" VALUE "ProductName", "Git\0" VALUE "ProductVersion", "@GIT_VERSION@\0" + VALUE "FileVersion", "@GIT_VERSION@\0" END END From dd73abf0a71f776a12c9a7d419dde99be27ccb53 Mon Sep 17 00:00:00 2001 From: Andrey Zabavnikov Date: Fri, 28 Oct 2022 17:12:06 +0300 Subject: [PATCH 062/229] status: fix for old-style submodules with commondir In f9b7573f6b00 (repository: free fields before overwriting them, 2017-09-05), Git was taught to release memory before overwriting it, but 357a03ebe9e0 (repository.c: move env-related setup code back to environment.c, 2018-03-03) changed the code so that it would not _always_ be overwritten. As a consequence, the `commondir` attribute would point to already-free()d memory. This seems not to cause problems in core Git, but there are add-on patches in Git for Windows where the `commondir` attribute is subsequently used and causing invalid memory accesses e.g. in setups containing old-style submodules (i.e. the ones with a `.git` directory within theirs worktrees) that have `commondir` configured. This fixes https://github.com/git-for-windows/git/pull/4083. Signed-off-by: Andrey Zabavnikov Signed-off-by: Johannes Schindelin --- repository.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository.c b/repository.c index ad02e4b29e89af..07326153f2ce70 100644 --- a/repository.c +++ b/repository.c @@ -156,7 +156,7 @@ static void repo_set_commondir(struct repository *repo, { struct strbuf sb = STRBUF_INIT; - free(repo->commondir); + FREE_AND_NULL(repo->commondir); if (commondir) { repo->different_commondir = 1; From c3cbbb458973080d52a370ec9b9a2bc73e5c6bf8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 21 Nov 2025 15:15:42 +0100 Subject: [PATCH 063/229] max_tree_depth: lower it for clang builds in general on Windows In 436a42215e5 (max_tree_depth: lower it for clangarm64 on Windows, 2025-04-23), I provided a work-around for a nasty issue with clangarm builds, where the stack is exhausted before the maximal tree depth is reached, and the resulting error cannot easily be handled by Git (because it would require Windows-specific handling). Turns out that this is not at all limited to ARM64. In my tests with CLANG64 in MSYS2 on the GitHub Actions runners, the test t6700.4 failed in the exact same way. What's worse: The limit needs to be quite a bit lower for x86_64 than for aarch64. In aforementioned tests, the breaking point was 1232: With 1231 it still worked as expected, with 1232 it would fail with the `STATUS_STACK_OVERFLOW` incorrectly mapped to exit code 127. For comparison, in my tests on GitHub Actions' Windows/ARM64 runners, the breaking point was 1439 instead. Therefore the condition needs to be adapted once more, to accommodate (with some safety margin) both aarch64 and x86_64 in clang-based builds on Windows, to let that test pass. Signed-off-by: Johannes Schindelin --- git-compat-util.h | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/git-compat-util.h b/git-compat-util.h index a0f901ce793ed6..7d1df43e212ae7 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -601,17 +601,23 @@ static inline bool strip_suffix(const char *str, const char *suffix, * the stack overflow can occur. */ #define DEFAULT_MAX_ALLOWED_TREE_DEPTH 512 -#elif defined(GIT_WINDOWS_NATIVE) && defined(__clang__) && defined(__aarch64__) +#elif defined(GIT_WINDOWS_NATIVE) && defined(__clang__) /* - * Similar to Visual C, it seems that on Windows/ARM64 the clang-based - * builds have a smaller stack space available. When running out of - * that stack space, a `STATUS_STACK_OVERFLOW` is produced. When the + * Similar to Visual C, it seems that clang-based builds on Windows + * have a smaller stack space available. When running out of that + * stack space, a `STATUS_STACK_OVERFLOW` is produced. When the * Git command was run from an MSYS2 Bash, this unfortunately results * in an exit code 127. Let's prevent that by lowering the maximal - * tree depth; This value seems to be low enough. + * tree depth; Unfortunately, it seems that the exact limit differs + * for aarch64 vs x86_64, and the difference is too large to simply + * use a single limit. */ +#if defined(__aarch64__) #define DEFAULT_MAX_ALLOWED_TREE_DEPTH 1280 #else +#define DEFAULT_MAX_ALLOWED_TREE_DEPTH 1152 +#endif +#else #define DEFAULT_MAX_ALLOWED_TREE_DEPTH 2048 #endif From 8b3ca322b36ba73740166eb9f769c223adc198d0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 25 Aug 2020 12:13:26 +0200 Subject: [PATCH 064/229] mingw: ignore HOMEDRIVE/HOMEPATH if it points to Windows' system directory Internally, Git expects the environment variable `HOME` to be set, and to point to the current user's home directory. This environment variable is not set by default on Windows, and therefore Git tries its best to construct one if it finds `HOME` unset. There are actually two different approaches Git tries: first, it looks at `HOMEDRIVE`/`HOMEPATH` because this is widely used in corporate environments with roaming profiles, and a user generally wants their global Git settings to be in a roaming profile. Only when `HOMEDRIVE`/`HOMEPATH` is either unset or does not point to a valid location, Git will fall back to using `USERPROFILE` instead. However, starting with Windows Vista, for secondary logons and services, the environment variables `HOMEDRIVE`/`HOMEPATH` point to Windows' system directory (usually `C:\Windows\system32`). That is undesirable, and that location is usually write-protected anyway. So let's verify that the `HOMEDRIVE`/`HOMEPATH` combo does not point to Windows' system directory before using it, falling back to `USERPROFILE` if it does. This fixes git-for-windows#2709 Initial-Path-by: Ivan Pozdeev Signed-off-by: Johannes Schindelin --- compat/mingw.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..93eb3509544092 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3196,6 +3196,18 @@ static size_t append_system_bin_dirs(char *path, size_t size) #endif #endif +static int is_system32_path(const char *path) +{ + WCHAR system32[MAX_PATH], wpath[MAX_PATH]; + + if (xutftowcs_path(wpath, path) < 0 || + !GetSystemDirectoryW(system32, ARRAY_SIZE(system32)) || + _wcsicmp(system32, wpath)) + return 0; + + return 1; +} + static void setup_windows_environment(void) { char *tmp = getenv("TMPDIR"); @@ -3236,7 +3248,8 @@ static void setup_windows_environment(void) strbuf_addstr(&buf, tmp); if ((tmp = getenv("HOMEPATH"))) { strbuf_addstr(&buf, tmp); - if (is_directory(buf.buf)) + if (!is_system32_path(buf.buf) && + is_directory(buf.buf)) setenv("HOME", buf.buf, 1); else tmp = NULL; /* use $USERPROFILE */ From f359f8c40134efafe5b3606ac7b9cb0c05e0d4ff Mon Sep 17 00:00:00 2001 From: Yuyi Wang Date: Sat, 11 Mar 2023 17:51:18 +0800 Subject: [PATCH 065/229] cmake: install headless-git. headless-git is a git executable without opening a console window. It is useful when other GUI executables want to call git. We should install it together with git on Windows. Signed-off-by: Yuyi Wang --- contrib/buildsystems/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index 9415c476ba4e85..d494d91e67c9f3 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -748,6 +748,7 @@ if(WIN32) endif() add_executable(headless-git ${CMAKE_SOURCE_DIR}/compat/win32/headless.c) + list(APPEND PROGRAMS_BUILT headless-git) if(CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_C_COMPILER_ID STREQUAL "Clang") target_link_options(headless-git PUBLIC -municode -Wl,-subsystem,windows) elseif(CMAKE_C_COMPILER_ID STREQUAL "MSVC") @@ -948,7 +949,7 @@ list(TRANSFORM git_perl_scripts PREPEND "${CMAKE_BINARY_DIR}/") #install foreach(program ${PROGRAMS_BUILT}) -if(program MATCHES "^(git|git-shell|scalar)$") +if(program MATCHES "^(git|git-shell|headless-git|scalar)$") install(TARGETS ${program} RUNTIME DESTINATION bin) else() From 88419824f7a4342e53ce5e5dbec988dba2753348 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20A=C3=9Fhauer?= Date: Sat, 2 Dec 2023 12:10:00 +0100 Subject: [PATCH 066/229] git.rc: include winuser.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit winuser.h contains the definition of RT_MANIFEST that our LLVM based toolchain needs to understand that we want to embed compat/win32/git.manifest as an application manifest. It currently just embeds it as additional data that Windows doesn't understand. This also helps our GCC based toolchain understand that we only want one copy embedded. It currently embeds one working assembly manifest and one nearly identical, but useless copy as additional data. This also teaches our Visual Studio based buildsystems to pick up the manifest file from git.rc. This means we don't have to explicitly specify it in contrib/buildsystems/Generators/Vcxproj.pm anymore. Slightly counter-intuitively this also means we have to explicitly tell Cmake not to embed a default manifest. This fixes https://github.com/git-for-windows/git/issues/4707 Signed-off-by: Matthias Aßhauer Signed-off-by: Johannes Schindelin --- contrib/buildsystems/CMakeLists.txt | 1 + git.rc.in | 1 + 2 files changed, 2 insertions(+) diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index d494d91e67c9f3..01cb4353283dbc 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -208,6 +208,7 @@ if(CMAKE_C_COMPILER_ID STREQUAL "MSVC") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_BINARY_DIR}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_BINARY_DIR}) add_compile_options(/MP /std:c11) + add_link_options(/MANIFEST:NO) endif() #default behaviour diff --git a/git.rc.in b/git.rc.in index e69444eef3f0c5..1d5b627b610549 100644 --- a/git.rc.in +++ b/git.rc.in @@ -1,3 +1,4 @@ +#include 1 VERSIONINFO FILEVERSION @GIT_MAJOR_VERSION@,@GIT_MINOR_VERSION@,@GIT_MICRO_VERSION@,@GIT_PATCH_LEVEL@ PRODUCTVERSION @GIT_MAJOR_VERSION@,@GIT_MINOR_VERSION@,@GIT_MICRO_VERSION@,@GIT_PATCH_LEVEL@ From 590e43e0d9e4e0ce9b2120f387c611a49930c22e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 7 May 2023 22:43:37 +0200 Subject: [PATCH 067/229] mingw: do load libcurl dynamically by default This will help with Git for Windows' maintenance going forward: It allows Git for Windows to switch its primary libcurl to a variant without the OpenSSL backend, while still loading an alternate when setting `http.sslBackend = openssl`. This is necessary to avoid maintenance headaches with upgrading OpenSSL: its major version name is encoded in the shared library's file name and hence major version updates (temporarily) break libraries that are linked against the OpenSSL library. Signed-off-by: Johannes Schindelin --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 9a9e5065aba310..890050c898361e 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -734,6 +734,7 @@ ifeq ($(uname_S),MINGW) HAVE_PLATFORM_PROCINFO = YesPlease CSPRNG_METHOD = rtlgenrandom BASIC_LDFLAGS += -municode -Wl,--tsaware + LAZYLOAD_LIBCURL = YesDoThatPlease COMPAT_CFLAGS += -DNOGDI -Icompat -Icompat/win32 COMPAT_CFLAGS += -DSTRIP_EXTENSION=\".exe\" COMPAT_OBJS += compat/mingw.o compat/winansi.o \ From 79f3393ac2b2f34bb5de12cc8ea7082accfb7844 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 2 Nov 2022 16:23:58 +0100 Subject: [PATCH 068/229] Add a GitHub workflow to verify that Git/Scalar work in Nano Server In Git for Windows v2.39.0, we fixed a regression where `git.exe` would no longer work in Windows Nano Server (frequently used in Docker containers). This GitHub workflow can be used to verify manually that the Git/Scalar executables work in Nano Server. Signed-off-by: Johannes Schindelin --- .github/workflows/nano-server.yml | 76 +++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/nano-server.yml diff --git a/.github/workflows/nano-server.yml b/.github/workflows/nano-server.yml new file mode 100644 index 00000000000000..2e6da3ceea6f90 --- /dev/null +++ b/.github/workflows/nano-server.yml @@ -0,0 +1,76 @@ +name: Windows Nano Server tests + +on: + workflow_dispatch: + +env: + DEVELOPER: 1 + +jobs: + test-nano-server: + runs-on: windows-2022 + env: + WINDBG_DIR: "C:/Program Files (x86)/Windows Kits/10/Debuggers/x64" + IMAGE: mcr.microsoft.com/powershell:nanoserver-ltsc2022 + + steps: + - uses: actions/checkout@v7 + - uses: git-for-windows/setup-git-for-windows-sdk@v2 + - name: build Git + shell: bash + run: make -j15 + - name: pull nanoserver image + shell: bash + run: docker pull $IMAGE + - name: run nano-server test + shell: bash + run: | + docker run \ + --user "ContainerAdministrator" \ + -v "$WINDBG_DIR:C:/dbg" \ + -v "$(cygpath -aw /mingw64/bin):C:/mingw64-bin" \ + -v "$(cygpath -aw .):C:/test" \ + $IMAGE pwsh.exe -Command ' + # Extend the PATH to include the `.dll` files in /mingw64/bin/ + $env:PATH += ";C:\mingw64-bin" + + # For each executable to test pick some no-operation set of + # flags/subcommands or something that should quickly result in an + # error with known exit code that is not a negative 32-bit + # number, and set the expected return code appropriately. + # + # Only test executables that could be expected to run in a UI + # less environment. + # + # ( Executable path, arguments, expected return code ) + # also note space is required before close parenthesis (a + # powershell quirk when defining nested arrays like this) + + $executables_to_test = @( + ("C:\test\git.exe", "", 1 ), + ("C:\test\scalar.exe", "version", 0 ) + ) + + foreach ($executable in $executables_to_test) + { + Write-Output "Now testing $($executable[0])" + &$executable[0] $executable[1] + if ($LASTEXITCODE -ne $executable[2]) { + # if we failed, run the debugger to find out what function + # or DLL could not be found and then exit the script with + # failure The missing DLL or EXE will be referenced near + # the end of the output + + # Set a flag to have the debugger show loader stub + # diagnostics. This requires running as administrator, + # otherwise the flag will be ignored. + C:\dbg\gflags -i $executable[0] +SLS + + C:\dbg\cdb.exe -c "g" -c "q" $executable[0] $executable[1] + + exit 1 + } + } + + exit 0 + ' From 13d42fbf5e6beffcb442dcbf595fa5475a33baaf Mon Sep 17 00:00:00 2001 From: David Lomas Date: Fri, 28 Jul 2023 15:31:25 +0100 Subject: [PATCH 069/229] mingw: suggest `windows.appendAtomically` in more cases When running Git for Windows on a remote APFS filesystem, it would appear that the `mingw_open_append()`/`write()` combination would fail almost exactly like on some CIFS-mounted shares as had been reported in https://github.com/git-for-windows/git/issues/2753, albeit with a different `errno` value. Let's handle that `errno` value just the same, by suggesting to set `windows.appendAtomically=false`. Signed-off-by: David Lomas Signed-off-by: Johannes Schindelin --- compat/mingw.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 3126ad081a39b1..2528310e2d1de8 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1038,7 +1038,7 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) { ssize_t result = write(fd, buf, len); - if (result < 0 && (errno == EINVAL || errno == ENOSPC) && buf) { + if (result < 0 && (errno == EINVAL || errno == EBADF || errno == ENOSPC) && buf) { int orig = errno; /* check if fd is a pipe */ @@ -1064,7 +1064,7 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) } errno = orig; - } else if (orig == EINVAL) + } else if (orig == EINVAL || errno == EBADF) errno = EPIPE; else { DWORD buf_size; From 6ce1149f25430ede480db32ae5a0e91b09f141d3 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 08:55:03 -0400 Subject: [PATCH 070/229] survey: stub in new experimental 'git-survey' command Start work on a new 'git survey' command to scan the repository for monorepo performance and scaling problems. The goal is to measure the various known "dimensions of scale" and serve as a foundation for adding additional measurements as we learn more about Git monorepo scaling problems. The initial goal is to complement the scanning and analysis performed by the GO-based 'git-sizer' (https://github.com/github/git-sizer) tool. It is hoped that by creating a builtin command, we may be able to take advantage of internal Git data structures and code that is not accessible from GO to gain further insight into potential scaling problems. Co-authored-by: Derrick Stolee Signed-off-by: Jeff Hostetler Signed-off-by: Derrick Stolee --- .gitignore | 1 + Documentation/config.adoc | 2 + Documentation/config/survey.adoc | 11 +++++ Documentation/git-survey.adoc | 36 +++++++++++++++ Documentation/meson.build | 1 + Makefile | 1 + builtin.h | 1 + builtin/survey.c | 76 ++++++++++++++++++++++++++++++++ command-list.txt | 1 + git.c | 1 + meson.build | 1 + t/meson.build | 1 + t/t1517-outside-repo.sh | 2 +- t/t8100-git-survey.sh | 18 ++++++++ 14 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 Documentation/config/survey.adoc create mode 100644 Documentation/git-survey.adoc create mode 100644 builtin/survey.c create mode 100755 t/t8100-git-survey.sh diff --git a/.gitignore b/.gitignore index 6b3f79165c9dab..ecde344e7147dc 100644 --- a/.gitignore +++ b/.gitignore @@ -171,6 +171,7 @@ /git-submodule /git-submodule--helper /git-subtree +/git-survey /git-svn /git-switch /git-symbolic-ref diff --git a/Documentation/config.adoc b/Documentation/config.adoc index ceb13ade6e4fa2..758c31d38e813c 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -598,6 +598,8 @@ include::config/status.adoc[] include::config/submodule.adoc[] +include::config/survey.adoc[] + include::config/tag.adoc[] include::config/tar.adoc[] diff --git a/Documentation/config/survey.adoc b/Documentation/config/survey.adoc new file mode 100644 index 00000000000000..c1b0f852a1250e --- /dev/null +++ b/Documentation/config/survey.adoc @@ -0,0 +1,11 @@ +survey.*:: + These variables adjust the default behavior of the `git survey` + command. The intention is that this command could be run in the + background with these options. ++ +-- + verbose:: + This boolean value implies the `--[no-]verbose` option. + progress:: + This boolean value implies the `--[no-]progress` option. +-- diff --git a/Documentation/git-survey.adoc b/Documentation/git-survey.adoc new file mode 100644 index 00000000000000..5f8ec9bfea673b --- /dev/null +++ b/Documentation/git-survey.adoc @@ -0,0 +1,36 @@ +git-survey(1) +============= + +NAME +---- +git-survey - EXPERIMENTAL: Measure various repository dimensions of scale + +SYNOPSIS +-------- +[verse] +(EXPERIMENTAL!) 'git survey' + +DESCRIPTION +----------- + +Survey the repository and measure various dimensions of scale. + +As repositories grow to "monorepo" size, certain data shapes can cause +performance problems. `git-survey` attempts to measure and report on +known problem areas. + +OPTIONS +------- + +--progress:: + Show progress. This is automatically enabled when interactive. + +OUTPUT +------ + +By default, `git survey` will print information about the repository in a +human-readable format that includes overviews and tables. + +GIT +--- +Part of the linkgit:git[1] suite diff --git a/Documentation/meson.build b/Documentation/meson.build index 99cb1cf34afce6..4e1e872fde52a8 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -146,6 +146,7 @@ manpages = { 'git-status.adoc' : 1, 'git-stripspace.adoc' : 1, 'git-submodule.adoc' : 1, + 'git-survey.adoc' : 1, 'git-svn.adoc' : 1, 'git-switch.adoc' : 1, 'git-symbolic-ref.adoc' : 1, diff --git a/Makefile b/Makefile index 984559fd680f09..2bf46849813dc4 100644 --- a/Makefile +++ b/Makefile @@ -1515,6 +1515,7 @@ BUILTIN_OBJS += builtin/sparse-checkout.o BUILTIN_OBJS += builtin/stash.o BUILTIN_OBJS += builtin/stripspace.o BUILTIN_OBJS += builtin/submodule--helper.o +BUILTIN_OBJS += builtin/survey.o BUILTIN_OBJS += builtin/symbolic-ref.o BUILTIN_OBJS += builtin/tag.o BUILTIN_OBJS += builtin/unpack-file.o diff --git a/builtin.h b/builtin.h index 7e64da9f433836..44fae36b6ee67e 100644 --- a/builtin.h +++ b/builtin.h @@ -261,6 +261,7 @@ int cmd_sparse_checkout(int argc, const char **argv, const char *prefix, struct int cmd_status(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_stash(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_stripspace(int argc, const char **argv, const char *prefix, struct repository *repo); +int cmd_survey(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_submodule__helper(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_switch(int argc, const char **argv, const char *prefix, struct repository *repo); int cmd_symbolic_ref(int argc, const char **argv, const char *prefix, struct repository *repo); diff --git a/builtin/survey.c b/builtin/survey.c new file mode 100644 index 00000000000000..f43c9e405ca8be --- /dev/null +++ b/builtin/survey.c @@ -0,0 +1,76 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "builtin.h" +#include "config.h" +#include "environment.h" +#include "parse-options.h" + +static const char * const survey_usage[] = { + N_("(EXPERIMENTAL!) git survey "), + NULL, +}; + +struct survey_opts { + int verbose; + int show_progress; +}; + +struct survey_context { + struct repository *repo; + + /* Options that control what is done. */ + struct survey_opts opts; +}; + +static int survey_load_config_cb(const char *var, const char *value, + const struct config_context *cctx, void *pvoid) +{ + struct survey_context *ctx = pvoid; + + if (!strcmp(var, "survey.verbose")) { + ctx->opts.verbose = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "survey.progress")) { + ctx->opts.show_progress = git_config_bool(var, value); + return 0; + } + + return git_default_config(var, value, cctx, pvoid); +} + +static void survey_load_config(struct survey_context *ctx) +{ + repo_config(the_repository, survey_load_config_cb, ctx); +} + +int cmd_survey(int argc, const char **argv, const char *prefix, struct repository *repo) +{ + static struct survey_context ctx = { + .opts = { + .verbose = 0, + .show_progress = -1, /* defaults to isatty(2) */ + }, + }; + + static struct option survey_options[] = { + OPT__VERBOSE(&ctx.opts.verbose, N_("verbose output")), + OPT_BOOL(0, "progress", &ctx.opts.show_progress, N_("show progress")), + OPT_END(), + }; + + show_usage_with_options_if_asked(argc, argv, + survey_usage, survey_options); + + ctx.repo = repo; + + prepare_repo_settings(ctx.repo); + survey_load_config(&ctx); + + argc = parse_options(argc, argv, prefix, survey_options, survey_usage, 0); + + if (ctx.opts.show_progress < 0) + ctx.opts.show_progress = isatty(2); + + return 0; +} diff --git a/command-list.txt b/command-list.txt index 79516ef5dda5ce..2776eae51f23d6 100644 --- a/command-list.txt +++ b/command-list.txt @@ -193,6 +193,7 @@ git-stash mainporcelain git-status mainporcelain info git-stripspace purehelpers git-submodule mainporcelain +git-survey mainporcelain git-svn foreignscminterface git-switch mainporcelain history git-symbolic-ref plumbingmanipulators diff --git a/git.c b/git.c index c8df287d70228c..79f64bcad49291 100644 --- a/git.c +++ b/git.c @@ -665,6 +665,7 @@ static struct cmd_struct commands[] = { { "status", cmd_status, RUN_SETUP | NEED_WORK_TREE }, { "stripspace", cmd_stripspace }, { "submodule--helper", cmd_submodule__helper, RUN_SETUP }, + { "survey", cmd_survey, RUN_SETUP }, { "switch", cmd_switch, RUN_SETUP | NEED_WORK_TREE }, { "symbolic-ref", cmd_symbolic_ref, RUN_SETUP }, { "tag", cmd_tag, RUN_SETUP | DELAY_PAGER_CONFIG }, diff --git a/meson.build b/meson.build index a2d373b37352a3..cbfe7c37cb2c8a 100644 --- a/meson.build +++ b/meson.build @@ -716,6 +716,7 @@ builtin_sources = [ 'builtin/stash.c', 'builtin/stripspace.c', 'builtin/submodule--helper.c', + 'builtin/survey.c', 'builtin/symbolic-ref.c', 'builtin/tag.c', 'builtin/unpack-file.c', diff --git a/t/meson.build b/t/meson.build index ed252313c96c74..c3b48fe229ed06 100644 --- a/t/meson.build +++ b/t/meson.build @@ -1005,6 +1005,7 @@ integration_tests = [ 't8014-blame-ignore-fuzzy.sh', 't8015-blame-diff-algorithm.sh', 't8020-last-modified.sh', + 't8100-git-survey.sh', 't9001-send-email.sh', 't9002-column.sh', 't9003-help-autocorrect.sh', diff --git a/t/t1517-outside-repo.sh b/t/t1517-outside-repo.sh index d1e915786eadb9..efbac29c0e7075 100755 --- a/t/t1517-outside-repo.sh +++ b/t/t1517-outside-repo.sh @@ -134,7 +134,7 @@ do http-backend | http-fetch | http-push | init-db | \ mktag | p4 | p4.py | pickaxe | remote-ftp | remote-ftps | \ remote-http | remote-https | replay | send-email | \ - sh-i18n--envsubst | shell | show | stage | \ + sh-i18n--envsubst | shell | show | stage | survey | \ upload-archive--writer | upload-pack | whatchanged) h_expect_outcome=expect_failure all_expect_outcome=expect_failure diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh new file mode 100755 index 00000000000000..a51a3e766e8a34 --- /dev/null +++ b/t/t8100-git-survey.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +test_description='git survey' + +GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main +export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME + +TEST_PASSES_SANITIZE_LEAK=0 +export TEST_PASSES_SANITIZE_LEAK + +. ./test-lib.sh + +test_expect_success 'git survey -h shows experimental warning' ' + test_expect_code 0 git survey -h >usage && + test_grep "EXPERIMENTAL!" usage +' + +test_done From d660ebf942675484c17698c5bba9b9dde68373ad Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Mon, 29 Apr 2024 09:51:34 -0400 Subject: [PATCH 071/229] survey: add command line opts to select references By default we will scan all references in "refs/heads/", "refs/tags/" and "refs/remotes/". Add command line opts let the use ask for all refs or a subset of them and to include a detached HEAD. Signed-off-by: Jeff Hostetler Signed-off-by: Derrick Stolee Signed-off-by: Johannes Schindelin --- Documentation/git-survey.adoc | 34 +++++ builtin/survey.c | 248 ++++++++++++++++++++++++++++++++++ t/t8100-git-survey.sh | 9 ++ 3 files changed, 291 insertions(+) diff --git a/Documentation/git-survey.adoc b/Documentation/git-survey.adoc index 5f8ec9bfea673b..56060d14b5cfef 100644 --- a/Documentation/git-survey.adoc +++ b/Documentation/git-survey.adoc @@ -19,12 +19,46 @@ As repositories grow to "monorepo" size, certain data shapes can cause performance problems. `git-survey` attempts to measure and report on known problem areas. +Ref Selection and Reachable Objects +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this first analysis phase, `git survey` will iterate over the set of +requested branches, tags, and other refs and treewalk over all of the +reachable commits, trees, and blobs and generate various statistics. + OPTIONS ------- --progress:: Show progress. This is automatically enabled when interactive. +Ref Selection +~~~~~~~~~~~~~ + +The following options control the set of refs that `git survey` will examine. +By default, `git survey` will look at tags, local branches, and remote refs. +If any of the following options are given, the default set is cleared and +only refs for the given options are added. + +--all-refs:: + Use all refs. This includes local branches, tags, remote refs, + notes, and stashes. This option overrides all of the following. + +--branches:: + Add local branches (`refs/heads/`) to the set. + +--tags:: + Add tags (`refs/tags/`) to the set. + +--remotes:: + Add remote branches (`refs/remote/`) to the set. + +--detached:: + Add HEAD to the set. + +--other:: + Add notes (`refs/notes/`) and stashes (`refs/stash/`) to the set. + OUTPUT ------ diff --git a/builtin/survey.c b/builtin/survey.c index f43c9e405ca8be..4cf289f1e59ec6 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -3,16 +3,55 @@ #include "builtin.h" #include "config.h" #include "environment.h" +#include "object.h" +#include "odb.h" #include "parse-options.h" +#include "progress.h" +#include "ref-filter.h" +#include "strvec.h" +#include "trace2.h" static const char * const survey_usage[] = { N_("(EXPERIMENTAL!) git survey "), NULL, }; +struct survey_refs_wanted { + int want_all_refs; /* special override */ + + int want_branches; + int want_tags; + int want_remotes; + int want_detached; + int want_other; /* see FILTER_REFS_OTHERS -- refs/notes/, refs/stash/ */ +}; + +static struct survey_refs_wanted default_ref_options = { + .want_all_refs = 1, +}; + struct survey_opts { int verbose; int show_progress; + struct survey_refs_wanted refs; +}; + +struct survey_report_ref_summary { + size_t refs_nr; + size_t branches_nr; + size_t remote_refs_nr; + size_t tags_nr; + size_t tags_annotated_nr; + size_t others_nr; + size_t unknown_nr; +}; + +/** + * This struct contains all of the information that needs to be printed + * at the end of the exploration of the repository and its references. + */ +struct survey_report { + struct survey_report_ref_summary refs; }; struct survey_context { @@ -20,8 +59,84 @@ struct survey_context { /* Options that control what is done. */ struct survey_opts opts; + + /* Info for output only. */ + struct survey_report report; + + /* + * The rest of the members are about enabling the activity + * of the 'git survey' command, including ref listings, object + * pointers, and progress. + */ + + struct progress *progress; + size_t progress_nr; + size_t progress_total; + + struct strvec refs; }; +static void clear_survey_context(struct survey_context *ctx) +{ + strvec_clear(&ctx->refs); +} + +/* + * After parsing the command line arguments, figure out which refs we + * should scan. + * + * If ANY were given in positive sense, then we ONLY include them and + * do not use the builtin values. + */ +static void fixup_refs_wanted(struct survey_context *ctx) +{ + struct survey_refs_wanted *rw = &ctx->opts.refs; + + /* + * `--all-refs` overrides and enables everything. + */ + if (rw->want_all_refs == 1) { + rw->want_branches = 1; + rw->want_tags = 1; + rw->want_remotes = 1; + rw->want_detached = 1; + rw->want_other = 1; + return; + } + + /* + * If none of the `--` were given, we assume all + * of the builtin unspecified values. + */ + if (rw->want_branches == -1 && + rw->want_tags == -1 && + rw->want_remotes == -1 && + rw->want_detached == -1 && + rw->want_other == -1) { + *rw = default_ref_options; + return; + } + + /* + * Since we only allow positive boolean values on the command + * line, we will only have true values where they specified + * a `--`. + * + * So anything that still has an unspecified value should be + * set to false. + */ + if (rw->want_branches == -1) + rw->want_branches = 0; + if (rw->want_tags == -1) + rw->want_tags = 0; + if (rw->want_remotes == -1) + rw->want_remotes = 0; + if (rw->want_detached == -1) + rw->want_detached = 0; + if (rw->want_other == -1) + rw->want_other = 0; +} + static int survey_load_config_cb(const char *var, const char *value, const struct config_context *cctx, void *pvoid) { @@ -44,18 +159,146 @@ static void survey_load_config(struct survey_context *ctx) repo_config(the_repository, survey_load_config_cb, ctx); } +static void do_load_refs(struct survey_context *ctx, + struct ref_array *ref_array) +{ + struct ref_filter filter = REF_FILTER_INIT; + struct ref_sorting *sorting; + struct string_list sorting_options = STRING_LIST_INIT_DUP; + + string_list_append(&sorting_options, "objectname"); + sorting = ref_sorting_options(&sorting_options); + + if (ctx->opts.refs.want_detached) + strvec_push(&ctx->refs, "HEAD"); + + if (ctx->opts.refs.want_all_refs) { + strvec_push(&ctx->refs, "refs/"); + } else { + if (ctx->opts.refs.want_branches) + strvec_push(&ctx->refs, "refs/heads/"); + if (ctx->opts.refs.want_tags) + strvec_push(&ctx->refs, "refs/tags/"); + if (ctx->opts.refs.want_remotes) + strvec_push(&ctx->refs, "refs/remotes/"); + if (ctx->opts.refs.want_other) { + strvec_push(&ctx->refs, "refs/notes/"); + strvec_push(&ctx->refs, "refs/stash/"); + } + } + + filter.name_patterns = ctx->refs.v; + filter.ignore_case = 0; + filter.match_as_path = 1; + + if (ctx->opts.show_progress) { + ctx->progress_total = 0; + ctx->progress = start_progress(ctx->repo, + _("Scanning refs..."), 0); + } + + filter_refs(ref_array, &filter, FILTER_REFS_KIND_MASK); + + if (ctx->opts.show_progress) { + ctx->progress_total = ref_array->nr; + display_progress(ctx->progress, ctx->progress_total); + } + + ref_array_sort(sorting, ref_array); + + stop_progress(&ctx->progress); + ref_filter_clear(&filter); + ref_sorting_release(sorting); +} + +/* + * The REFS phase: + * + * Load the set of requested refs and assess them for scalablity problems. + * Use that set to start a treewalk to all reachable objects and assess + * them. + * + * This data will give us insights into the repository itself (the number + * of refs, the size and shape of the DAG, the number and size of the + * objects). + * + * Theoretically, this data is independent of the on-disk representation + * (e.g. independent of packing concerns). + */ +static void survey_phase_refs(struct survey_context *ctx) +{ + struct ref_array ref_array = { 0 }; + + trace2_region_enter("survey", "phase/refs", ctx->repo); + do_load_refs(ctx, &ref_array); + + ctx->report.refs.refs_nr = ref_array.nr; + for (int i = 0; i < ref_array.nr; i++) { + size_t size; + struct ref_array_item *item = ref_array.items[i]; + + switch (item->kind) { + case FILTER_REFS_TAGS: + ctx->report.refs.tags_nr++; + if (odb_read_object_info(ctx->repo->objects, + &item->objectname, + &size) == OBJ_TAG) + ctx->report.refs.tags_annotated_nr++; + break; + + case FILTER_REFS_BRANCHES: + ctx->report.refs.branches_nr++; + break; + + case FILTER_REFS_REMOTES: + ctx->report.refs.remote_refs_nr++; + break; + + case FILTER_REFS_OTHERS: + ctx->report.refs.others_nr++; + break; + + default: + ctx->report.refs.unknown_nr++; + break; + } + } + + trace2_region_leave("survey", "phase/refs", ctx->repo); + + ref_array_clear(&ref_array); +} + int cmd_survey(int argc, const char **argv, const char *prefix, struct repository *repo) { static struct survey_context ctx = { .opts = { .verbose = 0, .show_progress = -1, /* defaults to isatty(2) */ + + .refs.want_all_refs = -1, + + .refs.want_branches = -1, /* default these to undefined */ + .refs.want_tags = -1, + .refs.want_remotes = -1, + .refs.want_detached = -1, + .refs.want_other = -1, }, + .refs = STRVEC_INIT, }; static struct option survey_options[] = { OPT__VERBOSE(&ctx.opts.verbose, N_("verbose output")), OPT_BOOL(0, "progress", &ctx.opts.show_progress, N_("show progress")), + + OPT_BOOL_F(0, "all-refs", &ctx.opts.refs.want_all_refs, N_("include all refs"), PARSE_OPT_NONEG), + + OPT_BOOL_F(0, "branches", &ctx.opts.refs.want_branches, N_("include branches"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "tags", &ctx.opts.refs.want_tags, N_("include tags"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "remotes", &ctx.opts.refs.want_remotes, N_("include all remotes refs"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "detached", &ctx.opts.refs.want_detached, N_("include detached HEAD"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "other", &ctx.opts.refs.want_other, N_("include notes and stashes"), PARSE_OPT_NONEG), + OPT_END(), }; @@ -72,5 +315,10 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor if (ctx.opts.show_progress < 0) ctx.opts.show_progress = isatty(2); + fixup_refs_wanted(&ctx); + + survey_phase_refs(&ctx); + + clear_survey_context(&ctx); return 0; } diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh index a51a3e766e8a34..8685f1fbf7bced 100755 --- a/t/t8100-git-survey.sh +++ b/t/t8100-git-survey.sh @@ -15,4 +15,13 @@ test_expect_success 'git survey -h shows experimental warning' ' test_grep "EXPERIMENTAL!" usage ' +test_expect_success 'create a semi-interesting repo' ' + test_commit_bulk 10 +' + +test_expect_success 'git survey (default)' ' + git survey >out 2>err && + test_line_count = 0 err +' + test_done From 6a7ab2dbb6c6e0f76d56cef4e59e127726fed3bf Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sun, 1 Sep 2024 15:58:32 -0400 Subject: [PATCH 072/229] survey: start pretty printing data in table form When 'git survey' provides information to the user, this will be presented in one of two formats: plaintext and JSON. The JSON implementation will be delayed until the functionality is complete for the plaintext format. The most important parts of the plaintext format are headers specifying the different sections of the report and tables providing concreted data. Create a custom table data structure that allows specifying a list of strings for the row values. When printing the table, check each column for the maximum width so we can create a table of the correct size from the start. The table structure is designed to be flexible to the different kinds of output that will be implemented in future changes. Signed-off-by: Derrick Stolee --- Documentation/git-survey.adoc | 7 ++ builtin/survey.c | 157 ++++++++++++++++++++++++++++++++++ t/t8100-git-survey.sh | 18 +++- 3 files changed, 181 insertions(+), 1 deletion(-) diff --git a/Documentation/git-survey.adoc b/Documentation/git-survey.adoc index 56060d14b5cfef..120ecb9a4d49f2 100644 --- a/Documentation/git-survey.adoc +++ b/Documentation/git-survey.adoc @@ -65,6 +65,13 @@ OUTPUT By default, `git survey` will print information about the repository in a human-readable format that includes overviews and tables. +References Summary +~~~~~~~~~~~~~~~~~~ + +The references summary includes a count of each kind of reference, +including branches, remote refs, and tags (split by "all" and +"annotated"). + GIT --- Part of the linkgit:git[1] suite diff --git a/builtin/survey.c b/builtin/survey.c index 4cf289f1e59ec6..527913d8dbf476 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -8,6 +8,7 @@ #include "parse-options.h" #include "progress.h" #include "ref-filter.h" +#include "strbuf.h" #include "strvec.h" #include "trace2.h" @@ -81,6 +82,160 @@ static void clear_survey_context(struct survey_context *ctx) strvec_clear(&ctx->refs); } +struct survey_table { + const char *table_name; + struct strvec header; + struct strvec *rows; + size_t rows_nr; + size_t rows_alloc; +}; + +#define SURVEY_TABLE_INIT { \ + .header = STRVEC_INIT, \ +} + +static void clear_table(struct survey_table *table) +{ + strvec_clear(&table->header); + for (size_t i = 0; i < table->rows_nr; i++) + strvec_clear(&table->rows[i]); + free(table->rows); +} + +static void insert_table_rowv(struct survey_table *table, ...) +{ + va_list ap; + char *arg; + ALLOC_GROW(table->rows, table->rows_nr + 1, table->rows_alloc); + + memset(&table->rows[table->rows_nr], 0, sizeof(struct strvec)); + + va_start(ap, table); + while ((arg = va_arg(ap, char *))) + strvec_push(&table->rows[table->rows_nr], arg); + va_end(ap); + + table->rows_nr++; +} + +#define SECTION_SEGMENT "========================================" +#define SECTION_SEGMENT_LEN 40 +static const char *section_line = SECTION_SEGMENT + SECTION_SEGMENT + SECTION_SEGMENT + SECTION_SEGMENT; +static const size_t section_len = 4 * SECTION_SEGMENT_LEN; + +static void print_table_title(const char *name, size_t *widths, size_t nr) +{ + size_t width = 3 * (nr - 1); + + for (size_t i = 0; i < nr; i++) + width += widths[i]; + + if (width > section_len) + width = section_len; + + printf("\n%s\n%.*s\n", name, (int)width, section_line); +} + +static void print_row_plaintext(struct strvec *row, size_t *widths) +{ + static struct strbuf line = STRBUF_INIT; + strbuf_setlen(&line, 0); + + for (size_t i = 0; i < row->nr; i++) { + const char *str = row->v[i]; + size_t len = strlen(str); + if (i) + strbuf_add(&line, " | ", 3); + strbuf_addchars(&line, ' ', widths[i] - len); + strbuf_add(&line, str, len); + } + printf("%s\n", line.buf); +} + +static void print_divider_plaintext(size_t *widths, size_t nr) +{ + static struct strbuf line = STRBUF_INIT; + strbuf_setlen(&line, 0); + + for (size_t i = 0; i < nr; i++) { + if (i) + strbuf_add(&line, "-+-", 3); + strbuf_addchars(&line, '-', widths[i]); + } + printf("%s\n", line.buf); +} + +static void print_table_plaintext(struct survey_table *table) +{ + size_t *column_widths; + size_t columns_nr = table->header.nr; + CALLOC_ARRAY(column_widths, columns_nr); + + for (size_t i = 0; i < columns_nr; i++) { + column_widths[i] = strlen(table->header.v[i]); + + for (size_t j = 0; j < table->rows_nr; j++) { + size_t rowlen = strlen(table->rows[j].v[i]); + if (column_widths[i] < rowlen) + column_widths[i] = rowlen; + } + } + + print_table_title(table->table_name, column_widths, columns_nr); + print_row_plaintext(&table->header, column_widths); + print_divider_plaintext(column_widths, columns_nr); + + for (size_t j = 0; j < table->rows_nr; j++) + print_row_plaintext(&table->rows[j], column_widths); + + free(column_widths); +} + +static void survey_report_plaintext_refs(struct survey_context *ctx) +{ + struct survey_report_ref_summary *refs = &ctx->report.refs; + struct survey_table table = SURVEY_TABLE_INIT; + + table.table_name = _("REFERENCES SUMMARY"); + + strvec_push(&table.header, _("Ref Type")); + strvec_push(&table.header, _("Count")); + + if (ctx->opts.refs.want_all_refs || ctx->opts.refs.want_branches) { + char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->branches_nr); + insert_table_rowv(&table, _("Branches"), fmt, NULL); + free(fmt); + } + + if (ctx->opts.refs.want_all_refs || ctx->opts.refs.want_remotes) { + char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->remote_refs_nr); + insert_table_rowv(&table, _("Remote refs"), fmt, NULL); + free(fmt); + } + + if (ctx->opts.refs.want_all_refs || ctx->opts.refs.want_tags) { + char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->tags_nr); + insert_table_rowv(&table, _("Tags (all)"), fmt, NULL); + free(fmt); + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->tags_annotated_nr); + insert_table_rowv(&table, _("Tags (annotated)"), fmt, NULL); + free(fmt); + } + + print_table_plaintext(&table); + clear_table(&table); +} + +static void survey_report_plaintext(struct survey_context *ctx) +{ + printf("GIT SURVEY for \"%s\"\n", ctx->repo->worktree); + printf("-----------------------------------------------------\n"); + survey_report_plaintext_refs(ctx); +} + /* * After parsing the command line arguments, figure out which refs we * should scan. @@ -319,6 +474,8 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor survey_phase_refs(&ctx); + survey_report_plaintext(&ctx); + clear_survey_context(&ctx); return 0; } diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh index 8685f1fbf7bced..ff0d22b0b02af3 100755 --- a/t/t8100-git-survey.sh +++ b/t/t8100-git-survey.sh @@ -21,7 +21,23 @@ test_expect_success 'create a semi-interesting repo' ' test_expect_success 'git survey (default)' ' git survey >out 2>err && - test_line_count = 0 err + test_line_count = 0 err && + + tr , " " >expect <<-EOF && + GIT SURVEY for "$(pwd)" + ----------------------------------------------------- + + REFERENCES SUMMARY + ======================== + , Ref Type | Count + -----------------+------ + , Branches | 1 + Remote refs | 0 + Tags (all) | 0 + Tags (annotated) | 0 + EOF + + test_cmp expect out ' test_done From 58398a6d9f9ea3adc3c78f0a12dafcefe8c2dd0d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sun, 1 Sep 2024 20:33:47 -0400 Subject: [PATCH 073/229] survey: add object count summary At the moment, nothing is obvious about the reason for the use of the path-walk API, but this will become more prevelant in future iterations. For now, use the path-walk API to sum up the counts of each kind of object. For example, this is the reachable object summary output for my local repo: REACHABLE OBJECT SUMMARY ======================== Object Type | Count ------------+------- Tags | 1343 Commits | 179344 Trees | 314350 Blobs | 184030 Signed-off-by: Derrick Stolee --- Documentation/git-survey.adoc | 6 ++ builtin/survey.c | 130 ++++++++++++++++++++++++++++++++-- t/t8100-git-survey.sh | 23 ++++-- 3 files changed, 148 insertions(+), 11 deletions(-) diff --git a/Documentation/git-survey.adoc b/Documentation/git-survey.adoc index 120ecb9a4d49f2..44f3a0568b7697 100644 --- a/Documentation/git-survey.adoc +++ b/Documentation/git-survey.adoc @@ -72,6 +72,12 @@ The references summary includes a count of each kind of reference, including branches, remote refs, and tags (split by "all" and "annotated"). +Reachable Object Summary +~~~~~~~~~~~~~~~~~~~~~~~~ + +The reachable object summary shows the total number of each kind of Git +object, including tags, commits, trees, and blobs. + GIT --- Part of the linkgit:git[1] suite diff --git a/builtin/survey.c b/builtin/survey.c index 527913d8dbf476..d85d999f497bde 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -3,13 +3,19 @@ #include "builtin.h" #include "config.h" #include "environment.h" +#include "hex.h" #include "object.h" #include "odb.h" +#include "object-name.h" #include "parse-options.h" +#include "path-walk.h" #include "progress.h" #include "ref-filter.h" +#include "refs.h" +#include "revision.h" #include "strbuf.h" #include "strvec.h" +#include "tag.h" #include "trace2.h" static const char * const survey_usage[] = { @@ -47,12 +53,20 @@ struct survey_report_ref_summary { size_t unknown_nr; }; +struct survey_report_object_summary { + size_t commits_nr; + size_t tags_nr; + size_t trees_nr; + size_t blobs_nr; +}; + /** * This struct contains all of the information that needs to be printed * at the end of the exploration of the repository and its references. */ struct survey_report { struct survey_report_ref_summary refs; + struct survey_report_object_summary reachable_objects; }; struct survey_context { @@ -75,10 +89,12 @@ struct survey_context { size_t progress_total; struct strvec refs; + struct ref_array ref_array; }; static void clear_survey_context(struct survey_context *ctx) { + ref_array_clear(&ctx->ref_array); strvec_clear(&ctx->refs); } @@ -129,10 +145,14 @@ static const size_t section_len = 4 * SECTION_SEGMENT_LEN; static void print_table_title(const char *name, size_t *widths, size_t nr) { size_t width = 3 * (nr - 1); + size_t min_width = strlen(name); for (size_t i = 0; i < nr; i++) width += widths[i]; + if (width < min_width) + width = min_width; + if (width > section_len) width = section_len; @@ -229,11 +249,43 @@ static void survey_report_plaintext_refs(struct survey_context *ctx) clear_table(&table); } +static void survey_report_plaintext_reachable_object_summary(struct survey_context *ctx) +{ + struct survey_report_object_summary *objs = &ctx->report.reachable_objects; + struct survey_table table = SURVEY_TABLE_INIT; + char *fmt; + + table.table_name = _("REACHABLE OBJECT SUMMARY"); + + strvec_push(&table.header, _("Object Type")); + strvec_push(&table.header, _("Count")); + + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->tags_nr); + insert_table_rowv(&table, _("Tags"), fmt, NULL); + free(fmt); + + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->commits_nr); + insert_table_rowv(&table, _("Commits"), fmt, NULL); + free(fmt); + + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->trees_nr); + insert_table_rowv(&table, _("Trees"), fmt, NULL); + free(fmt); + + fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->blobs_nr); + insert_table_rowv(&table, _("Blobs"), fmt, NULL); + free(fmt); + + print_table_plaintext(&table); + clear_table(&table); +} + static void survey_report_plaintext(struct survey_context *ctx) { printf("GIT SURVEY for \"%s\"\n", ctx->repo->worktree); printf("-----------------------------------------------------\n"); survey_report_plaintext_refs(ctx); + survey_report_plaintext_reachable_object_summary(ctx); } /* @@ -382,15 +434,13 @@ static void do_load_refs(struct survey_context *ctx, */ static void survey_phase_refs(struct survey_context *ctx) { - struct ref_array ref_array = { 0 }; - trace2_region_enter("survey", "phase/refs", ctx->repo); - do_load_refs(ctx, &ref_array); + do_load_refs(ctx, &ctx->ref_array); - ctx->report.refs.refs_nr = ref_array.nr; - for (int i = 0; i < ref_array.nr; i++) { + ctx->report.refs.refs_nr = ctx->ref_array.nr; + for (int i = 0; i < ctx->ref_array.nr; i++) { size_t size; - struct ref_array_item *item = ref_array.items[i]; + struct ref_array_item *item = ctx->ref_array.items[i]; switch (item->kind) { case FILTER_REFS_TAGS: @@ -420,8 +470,72 @@ static void survey_phase_refs(struct survey_context *ctx) } trace2_region_leave("survey", "phase/refs", ctx->repo); +} + +static void increment_object_counts( + struct survey_report_object_summary *summary, + enum object_type type, + size_t nr) +{ + switch (type) { + case OBJ_COMMIT: + summary->commits_nr += nr; + break; - ref_array_clear(&ref_array); + case OBJ_TREE: + summary->trees_nr += nr; + break; + + case OBJ_BLOB: + summary->blobs_nr += nr; + break; + + case OBJ_TAG: + summary->tags_nr += nr; + break; + + default: + break; + } +} + +static int survey_objects_path_walk_fn(const char *path UNUSED, + struct oid_array *oids, + enum object_type type, + void *data) +{ + struct survey_context *ctx = data; + + increment_object_counts(&ctx->report.reachable_objects, + type, oids->nr); + + return 0; +} + +static void survey_phase_objects(struct survey_context *ctx) +{ + struct rev_info revs = REV_INFO_INIT; + struct path_walk_info info = PATH_WALK_INFO_INIT; + unsigned int add_flags = 0; + + trace2_region_enter("survey", "phase/objects", ctx->repo); + + info.revs = &revs; + info.path_fn = survey_objects_path_walk_fn; + info.path_fn_data = ctx; + + repo_init_revisions(ctx->repo, &revs, ""); + revs.tag_objects = 1; + + for (int i = 0; i < ctx->ref_array.nr; i++) { + struct ref_array_item *item = ctx->ref_array.items[i]; + add_pending_oid(&revs, NULL, &item->objectname, add_flags); + } + + walk_objects_by_path(&info); + + release_revisions(&revs); + trace2_region_leave("survey", "phase/objects", ctx->repo); } int cmd_survey(int argc, const char **argv, const char *prefix, struct repository *repo) @@ -474,6 +588,8 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor survey_phase_refs(&ctx); + survey_phase_objects(&ctx); + survey_report_plaintext(&ctx); clear_survey_context(&ctx); diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh index ff0d22b0b02af3..68968129cb3a76 100755 --- a/t/t8100-git-survey.sh +++ b/t/t8100-git-survey.sh @@ -16,11 +16,17 @@ test_expect_success 'git survey -h shows experimental warning' ' ' test_expect_success 'create a semi-interesting repo' ' - test_commit_bulk 10 + test_commit_bulk 10 && + git tag -a -m one one HEAD~5 && + git tag -a -m two two HEAD~3 && + git tag -a -m three three two && + git tag -a -m four four three && + git update-ref -d refs/tags/three && + git update-ref -d refs/tags/two ' test_expect_success 'git survey (default)' ' - git survey >out 2>err && + git survey --all-refs >out 2>err && test_line_count = 0 err && tr , " " >expect <<-EOF && @@ -33,8 +39,17 @@ test_expect_success 'git survey (default)' ' -----------------+------ , Branches | 1 Remote refs | 0 - Tags (all) | 0 - Tags (annotated) | 0 + Tags (all) | 2 + Tags (annotated) | 2 + + REACHABLE OBJECT SUMMARY + ======================== + Object Type | Count + ------------+------ + Tags | 4 + Commits | 10 + Trees | 10 + Blobs | 10 EOF test_cmp expect out From ed2289e880e7f44a3be239fe061df623be585cd4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 23:33:56 +0200 Subject: [PATCH 074/229] repo: split annotated tags out from total tag count in structure Mirror what git survey already reports: lightweight tags (pointing straight at a commit/tree/blob) and annotated tags (pointing at an OBJ_TAG that is itself stored as a separate object) are different things in many monorepo contexts, and one of the differences git survey users routinely care about. Add an annotated_tags counter to struct ref_stats, populate it in count_references() by peeking at the ref OID's object type, and expose it as a sub-row under Tags in the table output and as references.tags.annotated.count in the machine-readable formats. Step toward pivoting the standalone git survey command onto git repo structure; this fills the first of the four feature gaps documented in the assessment. Tests in t1901 widened to assert the new row and key. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/repo.c | 10 ++++++++++ t/t1901-repo-structure.sh | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/builtin/repo.c b/builtin/repo.c index 6414a91a1d54b7..a11ca9da2f7e22 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -472,6 +472,7 @@ struct ref_stats { size_t branches; size_t remotes; size_t tags; + size_t annotated_tags; size_t others; }; @@ -657,6 +658,8 @@ static void stats_table_setup_structure(struct stats_table *table, stats_table_count_addf(table, ref_total, " * %s", _("Count")); stats_table_count_addf(table, refs->branches, " * %s", _("Branches")); stats_table_count_addf(table, refs->tags, " * %s", _("Tags")); + stats_table_count_addf(table, refs->annotated_tags, + " * %s", _("Annotated")); stats_table_count_addf(table, refs->remotes, " * %s", _("Remotes")); stats_table_count_addf(table, refs->others, " * %s", _("Others")); @@ -840,6 +843,8 @@ static void structure_keyvalue_print(struct repo_structure *stats, stats->refs.branches, value_delim); print_keyvalue("references.tags.count", key_delim, stats->refs.tags, value_delim); + print_keyvalue("references.tags.annotated.count", key_delim, + stats->refs.annotated_tags, value_delim); print_keyvalue("references.remotes.count", key_delim, stats->refs.remotes, value_delim); print_keyvalue("references.others.count", key_delim, @@ -892,6 +897,7 @@ static void structure_keyvalue_print(struct repo_structure *stats, struct count_references_data { struct ref_stats *stats; struct rev_info *revs; + struct repository *repo; struct progress *progress; }; @@ -910,6 +916,9 @@ static int count_references(const struct reference *ref, void *cb_data) break; case FILTER_REFS_TAGS: stats->tags++; + if (odb_read_object_info(data->repo->objects, + ref->oid, NULL) == OBJ_TAG) + stats->annotated_tags++; break; case FILTER_REFS_OTHERS: stats->others++; @@ -938,6 +947,7 @@ static void structure_count_references(struct ref_stats *stats, struct count_references_data data = { .stats = stats, .revs = revs, + .repo = repo, }; if (show_progress) diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index 02cc2b594a48b5..90774786d90a9a 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -33,6 +33,7 @@ test_expect_success 'empty repository' ' | * Count | 0 | | * Branches | 0 | | * Tags | 0 | + | * Annotated | 0 | | * Remotes | 0 | | * Others | 0 | | | | @@ -97,6 +98,7 @@ test_expect_success SHA1 'repository with references and objects' ' | * Count | 4 | | * Branches | 1 | | * Tags | 1 | + | * Annotated | 1 | | * Remotes | 1 | | * Others | 1 | | | | @@ -150,11 +152,13 @@ test_expect_success SHA1 'lines and nul format' ' ( cd repo && test_commit_bulk 42 && + git tag lightweight-tag-is-not-counted-as-annotated && git tag -a foo -m bar && cat >expect <<-EOF && references.branches.count=1 - references.tags.count=1 + references.tags.count=2 + references.tags.annotated.count=1 references.remotes.count=0 references.others.count=0 objects.commits.count=42 From 0f4c34a6b33b0083b4ebf6d99386b7276ea349b3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 16:41:17 +0200 Subject: [PATCH 075/229] diff: stop truncating the deflated-binary-diff size on Windows Continue the size_t evacuation around large object handling: with deflate_it() and the locals around it widened, the cast_size_t_to_ulong() shim the prior delta_delta() widening had to leave behind in emit_binary_diff_body() goes away. deflate_it() is file-static; the only callers are the two in emit_binary_diff_body() already touched here. emit_diff_symbol() formats the resulting sizes via uintmax_t / %"PRIuMAX", so the diff output is not affected; only the per-process upper bound on a binary patch chunk that this function can address grows beyond 4 GiB on Windows. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- diff.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/diff.c b/diff.c index 345ce7f19b41b1..1cbd6c255e4538 100644 --- a/diff.c +++ b/diff.c @@ -3647,8 +3647,8 @@ static int checkdiff_consume(void *priv, char *line, unsigned long len) } static unsigned char *deflate_it(char *data, - unsigned long size, - unsigned long *result_size) + size_t size, + size_t *result_size) { size_t bound; unsigned char *deflated; @@ -3677,10 +3677,10 @@ static void emit_binary_diff_body(struct diff_options *o, void *delta; void *deflated; void *data; - unsigned long orig_size; - unsigned long delta_size; - unsigned long deflate_size; - unsigned long data_size; + size_t orig_size; + size_t delta_size; + size_t deflate_size; + size_t data_size; /* We could do deflated delta, or we could do just deflated two, * whichever is smaller. @@ -3688,11 +3688,9 @@ static void emit_binary_diff_body(struct diff_options *o, delta = NULL; deflated = deflate_it(two->ptr, two->size, &deflate_size); if (one->size && two->size) { - size_t delta_size_st = 0; delta = diff_delta(one->ptr, one->size, two->ptr, two->size, - &delta_size_st, deflate_size); - delta_size = cast_size_t_to_ulong(delta_size_st); + &delta_size, deflate_size); if (delta) { void *to_free = delta; orig_size = delta_size; From 15d8dafef49f2c9f32ec93572c890978aca7ae94 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sun, 1 Sep 2024 20:58:35 -0400 Subject: [PATCH 076/229] survey: summarize total sizes by object type Now that we have explored objects by count, we can expand that a bit more to summarize the data for the on-disk and inflated size of those objects. This information is helpful for diagnosing both why disk space (and perhaps clone or fetch times) is growing but also why certain operations are slow because the inflated size of the abstract objects that must be processed is so large. Note: zlib-ng is slightly more efficient even at those small sizes. Even between zlib versions, there are slight differences in compression. To accommodate for that in the tests, not the exact numbers but some rough approximations are validated (the test should validate `git survey`, after all, not zlib). Signed-off-by: Derrick Stolee Signed-off-by: Johannes Schindelin --- builtin/survey.c | 133 ++++++++++++++++++++++++++++++++++++++++++ t/t8100-git-survey.sh | 37 +++++++++++- 2 files changed, 169 insertions(+), 1 deletion(-) diff --git a/builtin/survey.c b/builtin/survey.c index d85d999f497bde..bd19a823b92d38 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -60,6 +60,19 @@ struct survey_report_object_summary { size_t blobs_nr; }; +/** + * For some category given by 'label', count the number of objects + * that match that label along with the on-disk size and the size + * after decompressing (both with delta bases and zlib). + */ +struct survey_report_object_size_summary { + char *label; + size_t nr; + size_t disk_size; + size_t inflated_size; + size_t num_missing; +}; + /** * This struct contains all of the information that needs to be printed * at the end of the exploration of the repository and its references. @@ -67,8 +80,16 @@ struct survey_report_object_summary { struct survey_report { struct survey_report_ref_summary refs; struct survey_report_object_summary reachable_objects; + + struct survey_report_object_size_summary *by_type; }; +#define REPORT_TYPE_COMMIT 0 +#define REPORT_TYPE_TREE 1 +#define REPORT_TYPE_BLOB 2 +#define REPORT_TYPE_TAG 3 +#define REPORT_TYPE_COUNT 4 + struct survey_context { struct repository *repo; @@ -280,12 +301,48 @@ static void survey_report_plaintext_reachable_object_summary(struct survey_conte clear_table(&table); } +static void survey_report_object_sizes(const char *title, + const char *categories, + struct survey_report_object_size_summary *summary, + size_t summary_nr) +{ + struct survey_table table = SURVEY_TABLE_INIT; + table.table_name = title; + + strvec_push(&table.header, categories); + strvec_push(&table.header, _("Count")); + strvec_push(&table.header, _("Disk Size")); + strvec_push(&table.header, _("Inflated Size")); + + for (size_t i = 0; i < summary_nr; i++) { + char *label_str = xstrdup(summary[i].label); + char *nr_str = xstrfmt("%"PRIuMAX, (uintmax_t)summary[i].nr); + char *disk_str = xstrfmt("%"PRIuMAX, (uintmax_t)summary[i].disk_size); + char *inflate_str = xstrfmt("%"PRIuMAX, (uintmax_t)summary[i].inflated_size); + + insert_table_rowv(&table, label_str, nr_str, + disk_str, inflate_str, NULL); + + free(label_str); + free(nr_str); + free(disk_str); + free(inflate_str); + } + + print_table_plaintext(&table); + clear_table(&table); +} + static void survey_report_plaintext(struct survey_context *ctx) { printf("GIT SURVEY for \"%s\"\n", ctx->repo->worktree); printf("-----------------------------------------------------\n"); survey_report_plaintext_refs(ctx); survey_report_plaintext_reachable_object_summary(ctx); + survey_report_object_sizes(_("TOTAL OBJECT SIZES BY TYPE"), + _("Object Type"), + ctx->report.by_type, + REPORT_TYPE_COUNT); } /* @@ -499,6 +556,69 @@ static void increment_object_counts( } } +static void increment_totals(struct survey_context *ctx, + struct oid_array *oids, + struct survey_report_object_size_summary *summary) +{ + for (size_t i = 0; i < oids->nr; i++) { + struct object_info oi = OBJECT_INFO_INIT; + unsigned oi_flags = OBJECT_INFO_FOR_PREFETCH; + size_t object_length = 0; + off_t disk_sizep = 0; + enum object_type type; + + oi.typep = &type; + oi.sizep = &object_length; + oi.disk_sizep = &disk_sizep; + + if (odb_read_object_info_extended(ctx->repo->objects, + &oids->oid[i], + &oi, oi_flags) < 0) { + summary->num_missing++; + } else { + summary->nr++; + summary->disk_size += disk_sizep; + summary->inflated_size += object_length; + } + } +} + +static void increment_object_totals(struct survey_context *ctx, + struct oid_array *oids, + enum object_type type) +{ + struct survey_report_object_size_summary *total; + struct survey_report_object_size_summary summary = { 0 }; + + increment_totals(ctx, oids, &summary); + + switch (type) { + case OBJ_COMMIT: + total = &ctx->report.by_type[REPORT_TYPE_COMMIT]; + break; + + case OBJ_TREE: + total = &ctx->report.by_type[REPORT_TYPE_TREE]; + break; + + case OBJ_BLOB: + total = &ctx->report.by_type[REPORT_TYPE_BLOB]; + break; + + case OBJ_TAG: + total = &ctx->report.by_type[REPORT_TYPE_TAG]; + break; + + default: + BUG("No other type allowed"); + } + + total->nr += summary.nr; + total->disk_size += summary.disk_size; + total->inflated_size += summary.inflated_size; + total->num_missing += summary.num_missing; +} + static int survey_objects_path_walk_fn(const char *path UNUSED, struct oid_array *oids, enum object_type type, @@ -508,10 +628,20 @@ static int survey_objects_path_walk_fn(const char *path UNUSED, increment_object_counts(&ctx->report.reachable_objects, type, oids->nr); + increment_object_totals(ctx, oids, type); return 0; } +static void initialize_report(struct survey_context *ctx) +{ + CALLOC_ARRAY(ctx->report.by_type, REPORT_TYPE_COUNT); + ctx->report.by_type[REPORT_TYPE_COMMIT].label = xstrdup(_("Commits")); + ctx->report.by_type[REPORT_TYPE_TREE].label = xstrdup(_("Trees")); + ctx->report.by_type[REPORT_TYPE_BLOB].label = xstrdup(_("Blobs")); + ctx->report.by_type[REPORT_TYPE_TAG].label = xstrdup(_("Tags")); +} + static void survey_phase_objects(struct survey_context *ctx) { struct rev_info revs = REV_INFO_INIT; @@ -524,12 +654,15 @@ static void survey_phase_objects(struct survey_context *ctx) info.path_fn = survey_objects_path_walk_fn; info.path_fn_data = ctx; + initialize_report(ctx); + repo_init_revisions(ctx->repo, &revs, ""); revs.tag_objects = 1; for (int i = 0; i < ctx->ref_array.nr; i++) { struct ref_array_item *item = ctx->ref_array.items[i]; add_pending_oid(&revs, NULL, &item->objectname, add_flags); + display_progress(ctx->progress, ++(ctx->progress_nr)); } walk_objects_by_path(&info); diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh index 68968129cb3a76..88d80597579603 100755 --- a/t/t8100-git-survey.sh +++ b/t/t8100-git-survey.sh @@ -25,10 +25,35 @@ test_expect_success 'create a semi-interesting repo' ' git update-ref -d refs/tags/two ' +approximate_sizes() { + # very simplistic approximate rounding + sed -Ee "s/ *(1[0-9][0-9])( |$)/ ~0.1kB\2/g" \ + -e "s/ *(4[6-9][0-9]|5[0-6][0-9])( |$)/ ~0.5kB\2/g" \ + -e "s/ *(5[6-9][0-9]|6[0-6][0-9])( |$)/ ~0.6kB\2/g" \ + -e "s/ *1(4[89][0-9]|5[0-8][0-9])( |$)/ ~1.5kB\2/g" \ + -e "s/ *1(69[0-9]|7[0-9][0-9])( |$)/ ~1.7kB\2/g" \ + -e "s/ *1(79[0-9]|8[0-9][0-9])( |$)/ ~1.8kB\2/g" \ + -e "s/ *2(1[0-9][0-9]|20[0-1])( |$)/ ~2.1kB\2/g" \ + -e "s/ *2(3[0-9][0-9]|4[0-1][0-9])( |$)/ ~2.3kB\2/g" \ + -e "s/ *2(5[0-9][0-9]|6[0-1][0-9])( |$)/ ~2.5kB\2/g" \ + "$@" +} + test_expect_success 'git survey (default)' ' git survey --all-refs >out 2>err && test_line_count = 0 err && + test_oid_cache <<-EOF && + commits_sizes sha1:~1.5kB | ~2.1kB + commits_sizes sha256:~1.8kB | ~2.5kB + trees_sizes sha1:~0.5kB | ~1.7kB + trees_sizes sha256:~0.6kB | ~2.3kB + blobs_sizes sha1:~0.1kB | ~0.1kB + blobs_sizes sha256:~0.1kB | ~0.1kB + tags_sizes sha1:~0.5kB | ~0.5kB + tags_sizes sha256:~0.5kB | ~0.6kB + EOF + tr , " " >expect <<-EOF && GIT SURVEY for "$(pwd)" ----------------------------------------------------- @@ -50,9 +75,19 @@ test_expect_success 'git survey (default)' ' Commits | 10 Trees | 10 Blobs | 10 + + TOTAL OBJECT SIZES BY TYPE + =============================================== + Object Type | Count | Disk Size | Inflated Size + ------------+-------+-----------+-------------- + Commits | 10 | $(test_oid commits_sizes) + Trees | 10 | $(test_oid trees_sizes) + Blobs | 10 | $(test_oid blobs_sizes) + Tags | 4 | $(test_oid tags_sizes) EOF - test_cmp expect out + approximate_sizes out >out-edited && + test_cmp expect out-edited ' test_done From b643b694cf28796214d5ecbe1ffe671d8b1a32d0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 6 Jun 2026 00:47:53 +0200 Subject: [PATCH 077/229] repo: filter the structure scope via --ref-filter= `git repo structure` walks every reference enumerated by `refs_for_each_ref()` and feeds each reference's tip into the path walk that produces the object counts. There is no way to scope the inquiry to a subset of refs, even though that is the most common need when an operator is investigating what part of the history is driving cost: only branches, only release tags, only one remote's view, etc. Add a single `--ref-filter=` option that, when given, restricts both the reference count and the object walk to refs whose full name matches one of the patterns. The option is repeatable; multiple patterns form a union, so `--ref-filter='refs/heads/*' --ref-filter='refs/tags/v*'` includes local branches and tags whose short name starts with `v`. Patterns use `wildmatch()` with `WM_PATHNAME` semantics so a `*` does not cross `/`, matching the convention used by `git for-each-ref` positional arguments. Choosing a single flexible filter, rather than a proliferation of per-kind flags like `--branches`, `--tags`, `--remotes`, keeps the option surface small and lets the same mechanism express narrow selections the per-kind flags could not, such as "only release tags" (`'refs/tags/v*'`) or "only one remote's branches" (`'refs/remotes/origin/*'`). Without `--ref-filter`, behaviour is unchanged: every ref `refs_for_each_ref()` enumerates contributes. Both the reference counter and the path-walk seeding (via `add_pending_oid()`) sit on the same callback, so an early return when no pattern matches naturally excludes a ref from both. No separate object-walk machinery is needed. Cover the two interesting code paths with tests in t1901: a single filter narrowing to branches, and two filters unioning to include both branches and tags. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- Documentation/git-repo.adoc | 19 +++++++++++++++++-- builtin/repo.c | 36 ++++++++++++++++++++++++++++++++++-- t/t1901-repo-structure.sh | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index ae13ba7d023146..7ccd4c82c382bc 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -10,7 +10,7 @@ SYNOPSIS [synopsis] git repo info [--format=(lines|nul) | -z] [--all | ...] git repo info --keys [--format=(lines|nul) | -z] -git repo structure [--format=(table|lines|nul) | -z] +git repo structure [] DESCRIPTION ----------- @@ -56,7 +56,7 @@ supported: `nul`::: Similar to `lines`, but using a _NUL_ character after each value. -`structure [--format=(table|lines|nul) | -z]`:: +`structure`:: Retrieve statistics about the current repository structure. The following kinds of information are reported: + @@ -66,6 +66,21 @@ supported: * Total disk size of reachable objects by type * Largest reachable objects in the repository by type + +By default every reference enumerated by `for-each-ref` contributes to +the counts and object walk. Use `--ref-filter` to narrow the scope. + + `--ref-filter=`;; + Only count references whose full name matches one of the + given s, and only seed the object walk from + those references. The option is repeatable; multiple + patterns form a union. Patterns use the same `wildmatch` + semantics as `git for-each-ref`'s positional arguments, + so `*` does not cross `/`. Examples: + `--ref-filter='refs/heads/*'` for local branches, + `--ref-filter='refs/tags/v*'` for release tags, + `--ref-filter='refs/remotes/origin/*'` for a single + remote's branches. ++ The output format can be chosen through the flag `--format`. Three formats are supported: + diff --git a/builtin/repo.c b/builtin/repo.c index a11ca9da2f7e22..e7ca4ea66bbcbc 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -22,13 +22,14 @@ #include "tree.h" #include "tree-walk.h" #include "utf8.h" +#include "wildmatch.h" #define REPO_INFO_USAGE \ "git repo info [--format=(lines|nul) | -z] [--all | ...]", \ "git repo info --keys [--format=(lines|nul) | -z]" #define REPO_STRUCTURE_USAGE \ - "git repo structure [--format=(table|lines|nul) | -z]" + "git repo structure []" static const char *const repo_usage[] = { REPO_INFO_USAGE, @@ -898,15 +899,38 @@ struct count_references_data { struct ref_stats *stats; struct rev_info *revs; struct repository *repo; + const struct string_list *filters; struct progress *progress; }; +static int ref_matches_any_filter(const char *refname, + const struct string_list *filters) +{ + if (!filters->nr) + return 1; + for (size_t i = 0, namelen = strlen(refname); i < filters->nr; i++) { + const char *p = filters->items[i].string; + size_t plen = strlen(p); + if (plen <= namelen && + !strncmp(refname, p, plen) && + (refname[plen] == '\0' || refname[plen] == '/' || + (plen && p[plen - 1] == '/'))) + return 1; + if (!wildmatch(p, refname, WM_PATHNAME)) + return 1; + } + return 0; +} + static int count_references(const struct reference *ref, void *cb_data) { struct count_references_data *data = cb_data; struct ref_stats *stats = data->stats; size_t ref_count; + if (!ref_matches_any_filter(ref->name, data->filters)) + return 0; + switch (ref_kind_from_refname(ref->name)) { case FILTER_REFS_BRANCHES: stats->branches++; @@ -942,12 +966,14 @@ static int count_references(const struct reference *ref, void *cb_data) static void structure_count_references(struct ref_stats *stats, struct rev_info *revs, struct repository *repo, + const struct string_list *filters, int show_progress) { struct count_references_data data = { .stats = stats, .revs = revs, .repo = repo, + .filters = filters, }; if (show_progress) @@ -1098,6 +1124,7 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, struct repo_structure stats = { 0 }; struct rev_info revs; int show_progress = -1; + struct string_list ref_filters = STRING_LIST_INIT_DUP; struct option options[] = { OPT_CALLBACK_F(0, "format", &format, N_("format"), N_("output format"), @@ -1107,6 +1134,9 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, PARSE_OPT_NONEG | PARSE_OPT_NOARG, parse_format_cb), OPT_BOOL(0, "progress", &show_progress, N_("show progress")), + OPT_STRING_LIST(0, "ref-filter", &ref_filters, N_("pattern"), + N_("only count refs matching ; " + "repeat to union multiple patterns")), OPT_END() }; @@ -1119,7 +1149,8 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, if (show_progress < 0) show_progress = isatty(2); - structure_count_references(&stats.refs, &revs, repo, show_progress); + structure_count_references(&stats.refs, &revs, repo, &ref_filters, + show_progress); structure_count_objects(&stats.objects, &revs, repo, show_progress); switch (format) { @@ -1138,6 +1169,7 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, } stats_table_clear(&table); + string_list_clear(&ref_filters, 0); release_revisions(&revs); return 0; diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index 90774786d90a9a..3d259eff00a715 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -228,6 +228,41 @@ test_expect_success 'progress meter option' ' ) ' +test_expect_success '--ref-filter narrows the set of refs' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit foo && + git tag v1 && + git update-ref refs/remotes/origin/main HEAD && + + git repo structure --format=lines \ + --ref-filter="refs/heads/*" >out && + test_grep "^references.branches.count=1$" out && + test_grep "^references.tags.count=0$" out && + test_grep "^references.remotes.count=0$" out + ) +' + +test_expect_success '--ref-filter unions multiple patterns' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit foo && + git tag v1 && + git update-ref refs/remotes/origin/main HEAD && + + git repo structure --format=lines \ + --ref-filter="refs/heads/*" \ + --ref-filter="refs/tags/*" >out && + test_grep "^references.branches.count=1$" out && + test_grep "^references.tags.count=2$" out && + test_grep "^references.remotes.count=0$" out + ) +' + test_expect_success 'git repo structure -h shows only repo structure usage' ' git repo structure -h >actual && test_grep "git repo structure" actual && From a3a07738525ab8195f3c2a96b427158f9925d7bf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 16:44:00 +0200 Subject: [PATCH 078/229] convert: widen gather_convert_stats() helpers to size_t Prep for the upcoming read_blob_data_from_index() widening, whose callers in convert.c feed the size they receive straight into these two helpers. Both are file-static, so the change is contained. Also fixes a small pre-existing narrowing on the get_wt_convert_stats_ascii() path, where strbuf.len (size_t) was passed to a unsigned long parameter. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- convert.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/convert.c b/convert.c index 036506842c3d41..74d452b0deac20 100644 --- a/convert.c +++ b/convert.c @@ -102,7 +102,7 @@ static int convert_is_binary(const struct text_stat *stats) return 0; } -static unsigned int gather_convert_stats(const char *data, unsigned long size) +static unsigned int gather_convert_stats(const char *data, size_t size) { struct text_stat stats; int ret = 0; @@ -119,7 +119,7 @@ static unsigned int gather_convert_stats(const char *data, unsigned long size) return ret; } -static const char *gather_convert_stats_ascii(const char *data, unsigned long size) +static const char *gather_convert_stats_ascii(const char *data, size_t size) { unsigned int convert_stats = gather_convert_stats(data, size); From d390cc801382240ea993664bc94a0c0c072dfdbb Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sun, 1 Sep 2024 21:21:54 -0400 Subject: [PATCH 079/229] survey: show progress during object walk Signed-off-by: Derrick Stolee --- builtin/survey.c | 16 ++++++++++++++++ t/t8100-git-survey.sh | 5 +++++ 2 files changed, 21 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index bd19a823b92d38..dd026e1bcdf414 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -630,6 +630,9 @@ static int survey_objects_path_walk_fn(const char *path UNUSED, type, oids->nr); increment_object_totals(ctx, oids, type); + ctx->progress_nr += oids->nr; + display_progress(ctx->progress, ctx->progress_nr); + return 0; } @@ -659,13 +662,26 @@ static void survey_phase_objects(struct survey_context *ctx) repo_init_revisions(ctx->repo, &revs, ""); revs.tag_objects = 1; + ctx->progress_nr = 0; + ctx->progress_total = ctx->ref_array.nr; + if (ctx->opts.show_progress) + ctx->progress = start_progress(ctx->repo, + _("Preparing object walk"), + ctx->progress_total); for (int i = 0; i < ctx->ref_array.nr; i++) { struct ref_array_item *item = ctx->ref_array.items[i]; add_pending_oid(&revs, NULL, &item->objectname, add_flags); display_progress(ctx->progress, ++(ctx->progress_nr)); } + stop_progress(&ctx->progress); + ctx->progress_nr = 0; + ctx->progress_total = 0; + if (ctx->opts.show_progress) + ctx->progress = start_progress(ctx->repo, + _("Walking objects"), 0); walk_objects_by_path(&info); + stop_progress(&ctx->progress); release_revisions(&revs); trace2_region_leave("survey", "phase/objects", ctx->repo); diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh index 88d80597579603..b63ba3a1c8e310 100755 --- a/t/t8100-git-survey.sh +++ b/t/t8100-git-survey.sh @@ -25,6 +25,11 @@ test_expect_success 'create a semi-interesting repo' ' git update-ref -d refs/tags/two ' +test_expect_success 'git survey --progress' ' + GIT_PROGRESS_DELAY=0 git survey --all-refs --progress >out 2>err && + grep "Preparing object walk" err +' + approximate_sizes() { # very simplistic approximate rounding sed -Ee "s/ *(1[0-9][0-9])( |$)/ ~0.1kB\2/g" \ From f31140f4c64cf69d070b1038389414eb69bdad5b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 6 Jun 2026 00:05:16 +0200 Subject: [PATCH 080/229] repo: report top-N paths by count, disk, and inflated size in structure `git survey` distinguishes itself from `git repo structure` largely by its path-level reporting: in addition to whole-repo totals it lists the paths whose object histories dominate the repository, ranked by raw count, on-disk size, and inflated size, separately for trees and blobs. That is often the most actionable output from `git survey`, since it points an operator at the directories and files that should be reviewed for cleanup, sparse-checkout exclusion, or rewriting. `git repo structure` already drives the same path-walk traversal that `git survey` uses to gather its per-path numbers; the callback simply discards the path. Aggregate per-(path, type) summaries inside that existing callback and add a bounded, descending-sorted "top-N" table keyed by each of the three axes. Gate the feature behind a new `--top=` option, defaulting to 0, so unadorned invocations are unaffected and pay no extra work for the top-N tracking. Mirror the sort and eviction strategy from `builtin/survey.c`: keep an array of at most N entries sorted from largest to smallest, walk it from the bottom on each candidate, and shift entries down when a new one belongs. Compared to `builtin/survey.c`, drop the void-pointer indirection in the table data, type the comparator's arguments, and fold the trivial comparators into the `(a > b) - (a < b)` idiom. For the human-readable `table` output, extend the existing nested bullet layout with two new top-level sections, `* Top trees` and `* Top blobs`, each containing three sub-tables (`Top by count`, `Top by disk size`, `Top by inflated size`). The path becomes the row name and the relevant scalar becomes the value, reusing `stats_table_count_addf` and `stats_table_size_addf` so units and column alignment match the rest of the table. For the `lines`/`nul` key-value formats, emit one `objects..top.by_..path=` entry alongside an `objects..top.by_..=` entry per ranked path, so consumers can dispatch by axis without parsing the schema. The root tree's path is the empty string as produced by the path-walk machinery; preserve that as-is to stay faithful to the upstream representation rather than fabricating a placeholder. This is the first piece of folding `git survey`'s functionality into `git repo structure`. Subsequent commits will add the corresponding configuration knob and, eventually, turn `git survey` into a thin deprecated shim over `git repo structure`. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- Documentation/git-repo.adoc | 11 ++ builtin/repo.c | 250 +++++++++++++++++++++++++++++++++++- 2 files changed, 258 insertions(+), 3 deletions(-) diff --git a/Documentation/git-repo.adoc b/Documentation/git-repo.adoc index 7ccd4c82c382bc..0812d87b8e58b3 100644 --- a/Documentation/git-repo.adoc +++ b/Documentation/git-repo.adoc @@ -65,6 +65,8 @@ supported: * Total inflated size of reachable objects by type * Total disk size of reachable objects by type * Largest reachable objects in the repository by type +* Optionally, the top-_N_ largest paths by count, on-disk size, and + inflated size (see `--top` below) + By default every reference enumerated by `for-each-ref` contributes to the counts and object walk. Use `--ref-filter` to narrow the scope. @@ -80,6 +82,15 @@ the counts and object walk. Use `--ref-filter` to narrow the scope. `--ref-filter='refs/tags/v*'` for release tags, `--ref-filter='refs/remotes/origin/*'` for a single remote's branches. + + `--top=`;; + Also report the _n_ largest paths in the repository, + separately for trees and blobs and separately ranked by + object count, on-disk size, and inflated size. Defaults + to `0`, which suppresses the detail tables. The default + can also be set via the `repo.structure.top` configuration + variable; an explicit `--top=` on the command line + overrides the configured value. + The output format can be chosen through the flag `--format`. Three formats are supported: diff --git a/builtin/repo.c b/builtin/repo.c index e7ca4ea66bbcbc..30c1685a1e2e65 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -469,6 +469,40 @@ struct largest_objects { struct object_data tree_entries; }; +/* + * Per-path summary of all objects that share a given (path, type) under the + * path-walk traversal: the count of objects, their on-disk size, and their + * inflated size. + */ +struct path_size_summary { + char *path; + size_t nr; + size_t disk_size; + size_t inflated_size; +}; + +typedef int (*path_summary_cmp)(const struct path_size_summary *, + const struct path_size_summary *); + +/* + * A bounded, descending-sorted list of the largest summaries seen so far, + * with a fixed comparison function defining "largest". New summaries are + * inserted with maybe_insert_into_top_paths(); smaller ones fall off the + * end of the list. + */ +struct top_paths_table { + path_summary_cmp cmp_fn; + size_t nr; + size_t alloc; + struct path_size_summary *data; +}; + +struct top_paths { + struct top_paths_table by_count; + struct top_paths_table by_disk; + struct top_paths_table by_inflated; +}; + struct ref_stats { size_t branches; size_t remotes; @@ -489,6 +523,8 @@ struct object_stats { struct object_values inflated_sizes; struct object_values disk_sizes; struct largest_objects largest; + struct top_paths top_trees; + struct top_paths top_blobs; }; struct repo_structure { @@ -733,6 +769,41 @@ static void stats_table_setup_structure(struct stats_table *table, " * %s", _("Maximum size")); } +static void stats_table_add_top_paths(struct stats_table *table, + const struct top_paths *top, + const char *header) +{ + if (!top->by_count.nr && !top->by_disk.nr && !top->by_inflated.nr) + return; + + stats_table_addf(table, ""); + stats_table_addf(table, "* %s", header); + + stats_table_addf(table, " * %s", _("Top by count")); + for (size_t i = 0; i < top->by_count.nr; i++) + stats_table_count_addf(table, top->by_count.data[i].nr, + " * %s", top->by_count.data[i].path); + + stats_table_addf(table, " * %s", _("Top by disk size")); + for (size_t i = 0; i < top->by_disk.nr; i++) + stats_table_size_addf(table, top->by_disk.data[i].disk_size, + " * %s", top->by_disk.data[i].path); + + stats_table_addf(table, " * %s", _("Top by inflated size")); + for (size_t i = 0; i < top->by_inflated.nr; i++) + stats_table_size_addf(table, + top->by_inflated.data[i].inflated_size, + " * %s", + top->by_inflated.data[i].path); +} + +static void stats_table_setup_top_paths(struct stats_table *table, + struct object_stats *objects) +{ + stats_table_add_top_paths(table, &objects->top_trees, _("Top trees")); + stats_table_add_top_paths(table, &objects->top_blobs, _("Top blobs")); +} + #define INDEX_WIDTH 4 static void stats_table_print_structure(const struct stats_table *table) @@ -837,6 +908,54 @@ static void print_object_data(const char *key, char key_delim, value_delim); } +static void print_keyvalue_path(const char *key, char key_delim, + const char *path, char value_delim) +{ + printf("%s%c", key, key_delim); + if (key_delim == '=') + quote_c_style(path, NULL, stdout, 0); + else + fputs(path, stdout); + fputc(value_delim, stdout); +} + +static void top_paths_keyvalue_print(const char *prefix, + const struct top_paths *top, + char key_delim, char value_delim) +{ + for (size_t i = 0; i < top->by_count.nr; i++) { + printf("%s.by_count.%" PRIuMAX ".", + prefix, (uintmax_t)(i + 1)); + print_keyvalue_path("path", key_delim, + top->by_count.data[i].path, value_delim); + printf("%s.by_count.%" PRIuMAX ".", + prefix, (uintmax_t)(i + 1)); + print_keyvalue("count", key_delim, + top->by_count.data[i].nr, value_delim); + } + for (size_t i = 0; i < top->by_disk.nr; i++) { + printf("%s.by_disk_size.%" PRIuMAX ".", + prefix, (uintmax_t)(i + 1)); + print_keyvalue_path("path", key_delim, + top->by_disk.data[i].path, value_delim); + printf("%s.by_disk_size.%" PRIuMAX ".", + prefix, (uintmax_t)(i + 1)); + print_keyvalue("disk_size", key_delim, + top->by_disk.data[i].disk_size, value_delim); + } + for (size_t i = 0; i < top->by_inflated.nr; i++) { + printf("%s.by_inflated_size.%" PRIuMAX ".", + prefix, (uintmax_t)(i + 1)); + print_keyvalue_path("path", key_delim, + top->by_inflated.data[i].path, value_delim); + printf("%s.by_inflated_size.%" PRIuMAX ".", + prefix, (uintmax_t)(i + 1)); + print_keyvalue("inflated_size", key_delim, + top->by_inflated.data[i].inflated_size, + value_delim); + } +} + static void structure_keyvalue_print(struct repo_structure *stats, char key_delim, char value_delim) { @@ -892,6 +1011,11 @@ static void structure_keyvalue_print(struct repo_structure *stats, print_object_data("objects.trees.max_entries", key_delim, &stats->objects.largest.tree_entries, value_delim); + top_paths_keyvalue_print("objects.trees.top", &stats->objects.top_trees, + key_delim, value_delim); + top_paths_keyvalue_print("objects.blobs.top", &stats->objects.top_blobs, + key_delim, value_delim); + fflush(stdout); } @@ -988,8 +1112,88 @@ struct count_objects_data { struct object_database *odb; struct object_stats *stats; struct progress *progress; + size_t top_nr; }; +static int cmp_by_nr(const struct path_size_summary *s1, + const struct path_size_summary *s2) +{ + return (s1->nr > s2->nr) - (s1->nr < s2->nr); +} + +static int cmp_by_disk_size(const struct path_size_summary *s1, + const struct path_size_summary *s2) +{ + return (s1->disk_size > s2->disk_size) - + (s1->disk_size < s2->disk_size); +} + +static int cmp_by_inflated_size(const struct path_size_summary *s1, + const struct path_size_summary *s2) +{ + return (s1->inflated_size > s2->inflated_size) - + (s1->inflated_size < s2->inflated_size); +} + +static void init_top_paths_table(struct top_paths_table *top, size_t limit, + path_summary_cmp cmp) +{ + top->cmp_fn = cmp; + top->alloc = limit; + top->nr = 0; + CALLOC_ARRAY(top->data, limit); +} + +static void init_top_paths(struct top_paths *top, size_t limit) +{ + init_top_paths_table(&top->by_count, limit, cmp_by_nr); + init_top_paths_table(&top->by_disk, limit, cmp_by_disk_size); + init_top_paths_table(&top->by_inflated, limit, cmp_by_inflated_size); +} + +static void clear_top_paths_table(struct top_paths_table *top) +{ + for (size_t i = 0; i < top->nr; i++) + free(top->data[i].path); + free(top->data); +} + +static void clear_top_paths(struct top_paths *top) +{ + clear_top_paths_table(&top->by_count); + clear_top_paths_table(&top->by_disk); + clear_top_paths_table(&top->by_inflated); +} + +/* + * Insert 'summary' into 'top' if it ranks among the top alloc entries by the + * table's comparator. The list is kept sorted from largest (index 0) to + * smallest. If the table is already full, the smallest entry is evicted to + * make room. + */ +static void maybe_insert_into_top_paths(struct top_paths_table *top, + const struct path_size_summary *summary) +{ + size_t pos = top->nr; + + while (pos > 0 && top->cmp_fn(&top->data[pos - 1], summary) < 0) + pos--; + + if (pos >= top->alloc) + return; + + if (top->nr == top->alloc) + free(top->data[top->nr - 1].path); + else + top->nr++; + + for (size_t i = top->nr - 1; i > pos; i--) + top->data[i] = top->data[i - 1]; + + top->data[pos] = *summary; + top->data[pos].path = xstrdup(summary->path); +} + static void check_largest(struct object_data *data, struct object_id *oid, size_t value) { @@ -1013,11 +1217,12 @@ static size_t count_tree_entries(struct object *obj) return count; } -static int count_objects(const char *path UNUSED, struct oid_array *oids, +static int count_objects(const char *path, struct oid_array *oids, enum object_type type, void *cb_data) { struct count_objects_data *data = cb_data; struct object_stats *stats = data->stats; + struct path_size_summary summary = { .path = (char *)path }; size_t object_count; for (size_t i = 0; i < oids->nr; i++) { @@ -1043,6 +1248,10 @@ static int count_objects(const char *path UNUSED, struct oid_array *oids, obj = parse_object_buffer(the_repository, &oids->oid[i], type, inflated, content, &eaten); + summary.nr++; + summary.disk_size += disk; + summary.inflated_size += inflated; + switch (type) { case OBJ_TAG: stats->type_counts.tags++; @@ -1085,6 +1294,22 @@ static int count_objects(const char *path UNUSED, struct oid_array *oids, free(content); } + if (data->top_nr) { + struct top_paths *top = NULL; + + if (type == OBJ_TREE) + top = &stats->top_trees; + else if (type == OBJ_BLOB) + top = &stats->top_blobs; + + if (top) { + maybe_insert_into_top_paths(&top->by_count, &summary); + maybe_insert_into_top_paths(&top->by_disk, &summary); + maybe_insert_into_top_paths(&top->by_inflated, + &summary); + } + } + object_count = get_total_object_values(&stats->type_counts); display_progress(data->progress, object_count); @@ -1093,12 +1318,14 @@ static int count_objects(const char *path UNUSED, struct oid_array *oids, static void structure_count_objects(struct object_stats *stats, struct rev_info *revs, - struct repository *repo, int show_progress) + struct repository *repo, size_t top_nr, + int show_progress) { struct path_walk_info info = PATH_WALK_INFO_INIT; struct count_objects_data data = { .odb = repo->objects, .stats = stats, + .top_nr = top_nr, }; info.revs = revs; @@ -1124,6 +1351,7 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, struct repo_structure stats = { 0 }; struct rev_info revs; int show_progress = -1; + int top_nr = 0; struct string_list ref_filters = STRING_LIST_INIT_DUP; struct option options[] = { OPT_CALLBACK_F(0, "format", &format, N_("format"), @@ -1137,25 +1365,37 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, OPT_STRING_LIST(0, "ref-filter", &ref_filters, N_("pattern"), N_("only count refs matching ; " "repeat to union multiple patterns")), + OPT_INTEGER(0, "top", &top_nr, + N_("report the top largest paths " + "per category")), OPT_END() }; argc = parse_options(argc, argv, prefix, options, repo_structure_usage, 0); if (argc) usage(_("too many arguments")); + if (top_nr < 0) + die(_("--top= must be non-negative")); repo_init_revisions(repo, &revs, prefix); if (show_progress < 0) show_progress = isatty(2); + if (top_nr) { + init_top_paths(&stats.objects.top_trees, top_nr); + init_top_paths(&stats.objects.top_blobs, top_nr); + } + structure_count_references(&stats.refs, &revs, repo, &ref_filters, show_progress); - structure_count_objects(&stats.objects, &revs, repo, show_progress); + structure_count_objects(&stats.objects, &revs, repo, top_nr, + show_progress); switch (format) { case FORMAT_TABLE: stats_table_setup_structure(&table, &stats); + stats_table_setup_top_paths(&table, &stats.objects); stats_table_print_structure(&table); break; case FORMAT_NEWLINE_TERMINATED: @@ -1170,6 +1410,10 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, stats_table_clear(&table); string_list_clear(&ref_filters, 0); + if (top_nr) { + clear_top_paths(&stats.objects.top_trees); + clear_top_paths(&stats.objects.top_blobs); + } release_revisions(&revs); return 0; From c0c71f142ee822c6b42a7ec41b50b0a5d0e2bdd6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 14 Nov 2019 20:09:23 +0100 Subject: [PATCH 081/229] mingw: make sure `errno` is set correctly when socket operations fail The winsock2 library provides functions that work on different data types than file descriptors, therefore we wrap them. But that is not the only difference: they also do not set `errno` but expect the callers to enquire about errors via `WSAGetLastError()`. Let's translate that into appropriate `errno` values whenever the socket operations fail so that Git's code base does not have to change its expectations. This closes https://github.com/git-for-windows/git/issues/2404 Helped-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/mingw.c | 157 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 147 insertions(+), 10 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..3b788764946ec0 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2419,18 +2419,150 @@ static void ensure_socket_initialization(void) initialized = 1; } +static int winsock_error_to_errno(DWORD err) +{ + switch (err) { + case WSAEINTR: return EINTR; + case WSAEBADF: return EBADF; + case WSAEACCES: return EACCES; + case WSAEFAULT: return EFAULT; + case WSAEINVAL: return EINVAL; + case WSAEMFILE: return EMFILE; + case WSAEWOULDBLOCK: return EWOULDBLOCK; + case WSAEINPROGRESS: return EINPROGRESS; + case WSAEALREADY: return EALREADY; + case WSAENOTSOCK: return ENOTSOCK; + case WSAEDESTADDRREQ: return EDESTADDRREQ; + case WSAEMSGSIZE: return EMSGSIZE; + case WSAEPROTOTYPE: return EPROTOTYPE; + case WSAENOPROTOOPT: return ENOPROTOOPT; + case WSAEPROTONOSUPPORT: return EPROTONOSUPPORT; + case WSAEOPNOTSUPP: return EOPNOTSUPP; + case WSAEAFNOSUPPORT: return EAFNOSUPPORT; + case WSAEADDRINUSE: return EADDRINUSE; + case WSAEADDRNOTAVAIL: return EADDRNOTAVAIL; + case WSAENETDOWN: return ENETDOWN; + case WSAENETUNREACH: return ENETUNREACH; + case WSAENETRESET: return ENETRESET; + case WSAECONNABORTED: return ECONNABORTED; + case WSAECONNRESET: return ECONNRESET; + case WSAENOBUFS: return ENOBUFS; + case WSAEISCONN: return EISCONN; + case WSAENOTCONN: return ENOTCONN; + case WSAETIMEDOUT: return ETIMEDOUT; + case WSAECONNREFUSED: return ECONNREFUSED; + case WSAELOOP: return ELOOP; + case WSAENAMETOOLONG: return ENAMETOOLONG; + case WSAEHOSTUNREACH: return EHOSTUNREACH; + case WSAENOTEMPTY: return ENOTEMPTY; + /* No errno equivalent; default to EIO */ + case WSAESOCKTNOSUPPORT: + case WSAEPFNOSUPPORT: + case WSAESHUTDOWN: + case WSAETOOMANYREFS: + case WSAEHOSTDOWN: + case WSAEPROCLIM: + case WSAEUSERS: + case WSAEDQUOT: + case WSAESTALE: + case WSAEREMOTE: + case WSASYSNOTREADY: + case WSAVERNOTSUPPORTED: + case WSANOTINITIALISED: + case WSAEDISCON: + case WSAENOMORE: + case WSAECANCELLED: + case WSAEINVALIDPROCTABLE: + case WSAEINVALIDPROVIDER: + case WSAEPROVIDERFAILEDINIT: + case WSASYSCALLFAILURE: + case WSASERVICE_NOT_FOUND: + case WSATYPE_NOT_FOUND: + case WSA_E_NO_MORE: + case WSA_E_CANCELLED: + case WSAEREFUSED: + case WSAHOST_NOT_FOUND: + case WSATRY_AGAIN: + case WSANO_RECOVERY: + case WSANO_DATA: + case WSA_QOS_RECEIVERS: + case WSA_QOS_SENDERS: + case WSA_QOS_NO_SENDERS: + case WSA_QOS_NO_RECEIVERS: + case WSA_QOS_REQUEST_CONFIRMED: + case WSA_QOS_ADMISSION_FAILURE: + case WSA_QOS_POLICY_FAILURE: + case WSA_QOS_BAD_STYLE: + case WSA_QOS_BAD_OBJECT: + case WSA_QOS_TRAFFIC_CTRL_ERROR: + case WSA_QOS_GENERIC_ERROR: + case WSA_QOS_ESERVICETYPE: + case WSA_QOS_EFLOWSPEC: + case WSA_QOS_EPROVSPECBUF: + case WSA_QOS_EFILTERSTYLE: + case WSA_QOS_EFILTERTYPE: + case WSA_QOS_EFILTERCOUNT: + case WSA_QOS_EOBJLENGTH: + case WSA_QOS_EFLOWCOUNT: +#ifndef _MSC_VER + case WSA_QOS_EUNKNOWNPSOBJ: +#endif + case WSA_QOS_EPOLICYOBJ: + case WSA_QOS_EFLOWDESC: + case WSA_QOS_EPSFLOWSPEC: + case WSA_QOS_EPSFILTERSPEC: + case WSA_QOS_ESDMODEOBJ: + case WSA_QOS_ESHAPERATEOBJ: + case WSA_QOS_RESERVED_PETYPE: + default: return EIO; + } +} + +/* + * On Windows, `errno` is a global macro to a function call. + * This makes it difficult to debug and single-step our mappings. + */ +static inline void set_wsa_errno(void) +{ + DWORD wsa = WSAGetLastError(); + int e = winsock_error_to_errno(wsa); + errno = e; + +#ifdef DEBUG_WSA_ERRNO + fprintf(stderr, "winsock error: %d -> %d\n", wsa, e); + fflush(stderr); +#endif +} + +static inline int winsock_return(int ret) +{ + if (ret < 0) + set_wsa_errno(); + + return ret; +} + +#define WINSOCK_RETURN(x) do { return winsock_return(x); } while (0) + #undef gethostname int mingw_gethostname(char *name, int namelen) { - ensure_socket_initialization(); - return gethostname(name, namelen); + ensure_socket_initialization(); + WINSOCK_RETURN(gethostname(name, namelen)); } #undef gethostbyname struct hostent *mingw_gethostbyname(const char *host) { + struct hostent *ret; + ensure_socket_initialization(); - return gethostbyname(host); + + ret = gethostbyname(host); + if (!ret) + set_wsa_errno(); + + return ret; } #undef getaddrinfo @@ -2438,7 +2570,7 @@ int mingw_getaddrinfo(const char *node, const char *service, const struct addrinfo *hints, struct addrinfo **res) { ensure_socket_initialization(); - return getaddrinfo(node, service, hints, res); + WINSOCK_RETURN(getaddrinfo(node, service, hints, res)); } int mingw_socket(int domain, int type, int protocol) @@ -2458,7 +2590,7 @@ int mingw_socket(int domain, int type, int protocol) * in errno so that _if_ someone looks up the code somewhere, * then it is at least the number that are usually listed. */ - errno = WSAGetLastError(); + set_wsa_errno(); return -1; } /* convert into a file descriptor */ @@ -2474,35 +2606,35 @@ int mingw_socket(int domain, int type, int protocol) int mingw_connect(int sockfd, struct sockaddr *sa, size_t sz) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return connect(s, sa, sz); + WINSOCK_RETURN(connect(s, sa, sz)); } #undef bind int mingw_bind(int sockfd, struct sockaddr *sa, size_t sz) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return bind(s, sa, sz); + WINSOCK_RETURN(bind(s, sa, sz)); } #undef setsockopt int mingw_setsockopt(int sockfd, int lvl, int optname, void *optval, int optlen) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return setsockopt(s, lvl, optname, (const char*)optval, optlen); + WINSOCK_RETURN(setsockopt(s, lvl, optname, (const char*)optval, optlen)); } #undef shutdown int mingw_shutdown(int sockfd, int how) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return shutdown(s, how); + WINSOCK_RETURN(shutdown(s, how)); } #undef listen int mingw_listen(int sockfd, int backlog) { SOCKET s = (SOCKET)_get_osfhandle(sockfd); - return listen(s, backlog); + WINSOCK_RETURN(listen(s, backlog)); } #undef accept @@ -2513,6 +2645,11 @@ int mingw_accept(int sockfd1, struct sockaddr *sa, socklen_t *sz) SOCKET s1 = (SOCKET)_get_osfhandle(sockfd1); SOCKET s2 = accept(s1, sa, sz); + if (s2 == INVALID_SOCKET) { + set_wsa_errno(); + return -1; + } + /* convert into a file descriptor */ if ((sockfd2 = _open_osfhandle(s2, O_RDWR|O_BINARY)) < 0) { int err = errno; From da56e99019608295f87ae696a82132be409f5a86 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 17:00:12 +0200 Subject: [PATCH 082/229] read-cache: stop truncating index blob sizes on Windows Continue the size_t evacuation. read_blob_data_from_index() reads the blob through the size_t odb_read_object() API but writes the size back through an unsigned long out-parameter, silently truncating anything past 4 GiB on Windows. Widen the out-parameter, drop the cast_size_t_to_ulong() shim, and move the matching locals in the two convert.c callers and the one in attr.c. Their downstream consumers (gather_convert_stats() widened in the prior commit and read_attr_from_buf() already size_t) take the new type directly. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- attr.c | 2 +- convert.c | 4 ++-- read-cache-ll.h | 2 +- read-cache.c | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/attr.c b/attr.c index 0e63f1b6de8f53..1acb4ef430f5c9 100644 --- a/attr.c +++ b/attr.c @@ -793,7 +793,7 @@ static struct attr_stack *read_attr_from_index(struct index_state *istate, { struct attr_stack *stack = NULL; char *buf; - unsigned long size; + size_t size; int sparse_dir_pos = -1; if (!istate) diff --git a/convert.c b/convert.c index 74d452b0deac20..77f06fcfdba018 100644 --- a/convert.c +++ b/convert.c @@ -141,7 +141,7 @@ const char *get_cached_convert_stats_ascii(struct index_state *istate, const char *path) { const char *ret; - unsigned long sz; + size_t sz; void *data = read_blob_data_from_index(istate, path, &sz); ret = gather_convert_stats_ascii(data, sz); free(data); @@ -223,7 +223,7 @@ static void check_global_conv_flags_eol(const char *path, static int has_crlf_in_index(struct index_state *istate, const char *path) { - unsigned long sz; + size_t sz; void *data; const char *crp; int has_crlf = 0; diff --git a/read-cache-ll.h b/read-cache-ll.h index 066dd8bc3b8397..d279059b5fa428 100644 --- a/read-cache-ll.h +++ b/read-cache-ll.h @@ -416,7 +416,7 @@ int chmod_index_entry(struct index_state *, struct cache_entry *ce, char flip); int ce_same_name(const struct cache_entry *a, const struct cache_entry *b); void set_object_name_for_intent_to_add_entry(struct cache_entry *ce); int index_name_is_other(struct index_state *, const char *, int); -void *read_blob_data_from_index(struct index_state *, const char *, unsigned long *); +void *read_blob_data_from_index(struct index_state *, const char *, size_t *); /* do stat comparison even if CE_VALID is true */ #define CE_MATCH_IGNORE_VALID 01 diff --git a/read-cache.c b/read-cache.c index 00c9960c8e48a3..222148f387c95b 100644 --- a/read-cache.c +++ b/read-cache.c @@ -3488,7 +3488,7 @@ int index_name_is_other(struct index_state *istate, const char *name, } void *read_blob_data_from_index(struct index_state *istate, - const char *path, unsigned long *size) + const char *path, size_t *size) { int pos, len; size_t sz; @@ -3519,7 +3519,7 @@ void *read_blob_data_from_index(struct index_state *istate, return NULL; } if (size) - *size = cast_size_t_to_ulong(sz); + *size = sz; return data; } From b66323f4b12289a6cf83c0b5867274f56e47e0d7 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sun, 1 Sep 2024 22:35:06 -0400 Subject: [PATCH 083/229] survey: add ability to track prioritized lists In future changes, we will make use of these methods. The intention is to keep track of the top contributors according to some metric. We don't want to store all of the entries and do a sort at the end, so track a constant-size table and remove rows that get pushed out depending on the chosen sorting algorithm. Co-authored-by: Jeff Hostetler Signed-off-by; Jeff Hostetler Signed-off-by: Derrick Stolee --- builtin/survey.c | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index dd026e1bcdf414..c513b653f9604a 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -73,6 +73,119 @@ struct survey_report_object_size_summary { size_t num_missing; }; +typedef int (*survey_top_cmp)(void *v1, void *v2); + +MAYBE_UNUSED +static int cmp_by_nr(void *v1, void *v2) +{ + struct survey_report_object_size_summary *s1 = v1; + struct survey_report_object_size_summary *s2 = v2; + + if (s1->nr < s2->nr) + return -1; + if (s1->nr > s2->nr) + return 1; + return 0; +} + +MAYBE_UNUSED +static int cmp_by_disk_size(void *v1, void *v2) +{ + struct survey_report_object_size_summary *s1 = v1; + struct survey_report_object_size_summary *s2 = v2; + + if (s1->disk_size < s2->disk_size) + return -1; + if (s1->disk_size > s2->disk_size) + return 1; + return 0; +} + +MAYBE_UNUSED +static int cmp_by_inflated_size(void *v1, void *v2) +{ + struct survey_report_object_size_summary *s1 = v1; + struct survey_report_object_size_summary *s2 = v2; + + if (s1->inflated_size < s2->inflated_size) + return -1; + if (s1->inflated_size > s2->inflated_size) + return 1; + return 0; +} + +/** + * Store a list of "top" categories by some sorting function. When + * inserting a new category, reorder the list and free the one that + * got ejected (if any). + */ +struct survey_report_top_table { + const char *name; + survey_top_cmp cmp_fn; + size_t nr; + size_t alloc; + + /** + * 'data' stores an array of structs and must be cast into + * the proper array type before evaluating an index. + */ + void *data; +}; + +MAYBE_UNUSED +static void init_top_sizes(struct survey_report_top_table *top, + size_t limit, const char *name, + survey_top_cmp cmp) +{ + struct survey_report_object_size_summary *sz_array; + + top->name = name; + top->cmp_fn = cmp; + top->alloc = limit; + top->nr = 0; + + CALLOC_ARRAY(sz_array, limit); + top->data = sz_array; +} + +MAYBE_UNUSED +static void clear_top_sizes(struct survey_report_top_table *top) +{ + struct survey_report_object_size_summary *sz_array = top->data; + + for (size_t i = 0; i < top->nr; i++) + free(sz_array[i].label); + free(sz_array); +} + +MAYBE_UNUSED +static void maybe_insert_into_top_size(struct survey_report_top_table *top, + struct survey_report_object_size_summary *summary) +{ + struct survey_report_object_size_summary *sz_array = top->data; + size_t pos = top->nr; + + /* Compare against list from the bottom. */ + while (pos > 0 && top->cmp_fn(&sz_array[pos - 1], summary) < 0) + pos--; + + /* Not big enough! */ + if (pos >= top->alloc) + return; + + /* We need to shift the data. */ + if (top->nr == top->alloc) + free(sz_array[top->nr - 1].label); + else + top->nr++; + + for (size_t i = top->nr - 1; i > pos; i--) + memcpy(&sz_array[i], &sz_array[i - 1], sizeof(*sz_array)); + + memcpy(&sz_array[pos], summary, sizeof(*summary)); + sz_array[pos].label = xstrdup(summary->label); +} + /** * This struct contains all of the information that needs to be printed * at the end of the exploration of the repository and its references. From e53422ad83461cf22c8d20c0c3e808d92e3ef964 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 6 Jun 2026 00:13:46 +0200 Subject: [PATCH 084/229] t1901: cover the --top option of `git repo structure` The preceding commit added `--top=` to `git repo structure`, reporting the top-N paths per type ranked by count, on-disk size, and inflated size. Cover the three behaviors that matter for that option: * Without `--top`, the key-value output emits no `top.*` keys, so existing parsers stay unaffected. * `--top=N` produces exactly N ranked entries on each of the six `objects..top.by_` axes (count/disk_size/inflated_size crossed with trees/blobs), and a constructed input where one blob is several orders of magnitude bigger than the other lets us assert the ordering on the disk-size and inflated-size axes. * A negative `--top` is rejected with a non-zero exit and a message naming the constraint, so a typo cannot silently degrade into the default zero. Avoid grep patterns starting with `--`; grep would parse the leading double dash as an option terminator. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- t/t1901-repo-structure.sh | 50 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index 3d259eff00a715..baafd7fbe756ec 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -263,6 +263,56 @@ test_expect_success '--ref-filter unions multiple patterns' ' ) ' +test_expect_success '--top omitted: no top.* keys' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit foo && + + git repo structure --format=lines >out && + test_grep ! "\.top\." out + ) +' + +test_expect_success '--top=N reports the N largest paths per axis' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + mkdir -p dir1 dir2 && + echo small >dir1/small.txt && + printf "%010000d" 0 >dir2/big.txt && + git add . && + test_tick && + git commit -m commit && + + git repo structure --format=lines --top=2 >out && + + # Two ranked entries on each axis for both types. + for axis in by_count by_disk_size by_inflated_size + do + for type in trees blobs + do + key=objects.${type}.top.${axis} && + test_grep -E "^${key}\.1\.path=" out && + test_grep -E "^${key}\.2\.path=" out && + test_grep ! -E "^${key}\.3\." out || return 1 + done + done && + + # The big blob outranks the small one on disk and inflated. + key=objects.blobs.top && + test_grep "^${key}.by_disk_size.1.path=dir2/big.txt$" out && + test_grep "^${key}.by_inflated_size.1.path=dir2/big.txt$" out + ) +' + +test_expect_success '--top rejects negative values' ' + test_must_fail git repo structure --top=-1 2>err && + test_grep "must be non-negative" err +' + test_expect_success 'git repo structure -h shows only repo structure usage' ' git repo structure -h >actual && test_grep "git repo structure" actual && From dd1b3e523740f7d85097013ae654454e47d27640 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20A=C3=9Fhauer?= Date: Sun, 22 Dec 2024 17:15:39 +0100 Subject: [PATCH 085/229] compat/mingw: handle WSA errors in strerror MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We map WSAGetLastError() errors to errno errors in winsock_error_to_errno(), but the MSVC strerror() implementation only produces "Unknown error" for most of them. Produce some more meaningful error messages in these cases. Our builds for ARM64 link against the newer UCRT strerror() that does know these errors, so we won't change the strerror() used there. The wording of the messages is copied from glibc strerror() messages. Reported-by: M Hickford Signed-off-by: Matthias Aßhauer Signed-off-by: Johannes Schindelin --- Makefile | 1 + compat/mingw-posix.h | 5 +++ compat/mingw.c | 85 ++++++++++++++++++++++++++++++++++++++++++ t/meson.build | 1 + t/unit-tests/u-mingw.c | 72 +++++++++++++++++++++++++++++++++++ 5 files changed, 164 insertions(+) create mode 100644 t/unit-tests/u-mingw.c diff --git a/Makefile b/Makefile index 984559fd680f09..2ff4c8a35df31e 100644 --- a/Makefile +++ b/Makefile @@ -1554,6 +1554,7 @@ CLAR_TEST_SUITES += u-hash CLAR_TEST_SUITES += u-hashmap CLAR_TEST_SUITES += u-list-objects-filter-options CLAR_TEST_SUITES += u-mem-pool +CLAR_TEST_SUITES += u-mingw CLAR_TEST_SUITES += u-odb-inmemory CLAR_TEST_SUITES += u-oid-array CLAR_TEST_SUITES += u-oidmap diff --git a/compat/mingw-posix.h b/compat/mingw-posix.h index 2d989fd762474e..da934834a1a177 100644 --- a/compat/mingw-posix.h +++ b/compat/mingw-posix.h @@ -288,6 +288,11 @@ int mingw_socket(int domain, int type, int protocol); int mingw_connect(int sockfd, struct sockaddr *sa, size_t sz); #define connect mingw_connect +char *mingw_strerror(int errnum); +#ifndef _UCRT +#define strerror mingw_strerror +#endif + int mingw_bind(int sockfd, struct sockaddr *sa, size_t sz); #define bind mingw_bind diff --git a/compat/mingw.c b/compat/mingw.c index 3b788764946ec0..d0416f8d8e2664 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2544,6 +2544,91 @@ static inline int winsock_return(int ret) #define WINSOCK_RETURN(x) do { return winsock_return(x); } while (0) +#undef strerror +char *mingw_strerror(int errnum) +{ + static char buf[41] =""; + switch (errnum) { + case EWOULDBLOCK: + xsnprintf(buf, 41, "%s", "Operation would block"); + break; + case EINPROGRESS: + xsnprintf(buf, 41, "%s", "Operation now in progress"); + break; + case EALREADY: + xsnprintf(buf, 41, "%s", "Operation already in progress"); + break; + case ENOTSOCK: + xsnprintf(buf, 41, "%s", "Socket operation on non-socket"); + break; + case EDESTADDRREQ: + xsnprintf(buf, 41, "%s", "Destination address required"); + break; + case EMSGSIZE: + xsnprintf(buf, 41, "%s", "Message too long"); + break; + case EPROTOTYPE: + xsnprintf(buf, 41, "%s", "Protocol wrong type for socket"); + break; + case ENOPROTOOPT: + xsnprintf(buf, 41, "%s", "Protocol not available"); + break; + case EPROTONOSUPPORT: + xsnprintf(buf, 41, "%s", "Protocol not supported"); + break; + case EOPNOTSUPP: + xsnprintf(buf, 41, "%s", "Operation not supported"); + break; + case EAFNOSUPPORT: + xsnprintf(buf, 41, "%s", "Address family not supported by protocol"); + break; + case EADDRINUSE: + xsnprintf(buf, 41, "%s", "Address already in use"); + break; + case EADDRNOTAVAIL: + xsnprintf(buf, 41, "%s", "Cannot assign requested address"); + break; + case ENETDOWN: + xsnprintf(buf, 41, "%s", "Network is down"); + break; + case ENETUNREACH: + xsnprintf(buf, 41, "%s", "Network is unreachable"); + break; + case ENETRESET: + xsnprintf(buf, 41, "%s", "Network dropped connection on reset"); + break; + case ECONNABORTED: + xsnprintf(buf, 41, "%s", "Software caused connection abort"); + break; + case ECONNRESET: + xsnprintf(buf, 41, "%s", "Connection reset by peer"); + break; + case ENOBUFS: + xsnprintf(buf, 41, "%s", "No buffer space available"); + break; + case EISCONN: + xsnprintf(buf, 41, "%s", "Transport endpoint is already connected"); + break; + case ENOTCONN: + xsnprintf(buf, 41, "%s", "Transport endpoint is not connected"); + break; + case ETIMEDOUT: + xsnprintf(buf, 41, "%s", "Connection timed out"); + break; + case ECONNREFUSED: + xsnprintf(buf, 41, "%s", "Connection refused"); + break; + case ELOOP: + xsnprintf(buf, 41, "%s", "Too many levels of symbolic links"); + break; + case EHOSTUNREACH: + xsnprintf(buf, 41, "%s", "No route to host"); + break; + default: return strerror(errnum); + } + return buf; +} + #undef gethostname int mingw_gethostname(char *name, int namelen) { diff --git a/t/meson.build b/t/meson.build index ed252313c96c74..323f9142b3e679 100644 --- a/t/meson.build +++ b/t/meson.build @@ -6,6 +6,7 @@ clar_test_suites = [ 'unit-tests/u-hashmap.c', 'unit-tests/u-list-objects-filter-options.c', 'unit-tests/u-mem-pool.c', + 'unit-tests/u-mingw.c', 'unit-tests/u-odb-inmemory.c', 'unit-tests/u-oid-array.c', 'unit-tests/u-oidmap.c', diff --git a/t/unit-tests/u-mingw.c b/t/unit-tests/u-mingw.c new file mode 100644 index 00000000000000..cb74da5e793a33 --- /dev/null +++ b/t/unit-tests/u-mingw.c @@ -0,0 +1,72 @@ +#include "unit-test.h" + +#if defined(GIT_WINDOWS_NATIVE) && !defined(_UCRT) +#undef strerror +int errnos_contains(int); +static int errnos [53]={ + /* errnos in err_win_to_posix */ + EACCES, EBUSY, EEXIST, ERANGE, EIO, ENODEV, ENXIO, ENOEXEC, EINVAL, ENOENT, + EPIPE, ENAMETOOLONG, ENOSYS, ENOTEMPTY, ENOSPC, EFAULT, EBADF, EPERM, EINTR, + E2BIG, ESPIPE, ENOMEM, EXDEV, EAGAIN, ENFILE, EMFILE, ECHILD, EROFS, + /* errnos only in winsock_error_to_errno */ + EWOULDBLOCK, EINPROGRESS, EALREADY, ENOTSOCK, EDESTADDRREQ, EMSGSIZE, + EPROTOTYPE, ENOPROTOOPT, EPROTONOSUPPORT, EOPNOTSUPP, EAFNOSUPPORT, + EADDRINUSE, EADDRNOTAVAIL, ENETDOWN, ENETUNREACH, ENETRESET, ECONNABORTED, + ECONNRESET, ENOBUFS, EISCONN, ENOTCONN, ETIMEDOUT, ECONNREFUSED, ELOOP, + EHOSTUNREACH + }; + +int errnos_contains(int errnum) +{ + for(int i=0;i<53;i++) + if(errnos[i]==errnum) + return 1; + return 0; +} +#endif + +void test_mingw__no_strerror_shim_on_ucrt(void) +{ +#if defined(GIT_WINDOWS_NATIVE) && defined(_UCRT) + cl_assert_(strerror != mingw_strerror, + "mingw_strerror is unnescessary when building against UCRT"); +#else + cl_skip(); +#endif +} + +void test_mingw__strerror(void) +{ +#if defined(GIT_WINDOWS_NATIVE) && !defined(_UCRT) + for(int i=0;i<53;i++) + { + char *crt; + char *mingw; + mingw = mingw_strerror(errnos[i]); + crt = strerror(errnos[i]); + cl_assert_(!strcasestr(mingw, "unknown error"), + "mingw_strerror should know all errno values we care about"); + if(!strcasestr(crt, "unknown error")) + cl_assert_equal_s(crt,mingw); + } +#else + cl_skip(); +#endif +} + +void test_mingw__errno_translation(void) +{ +#if defined(GIT_WINDOWS_NATIVE) && !defined(_UCRT) + /* GetLastError() return values are currently defined from 0 to 15841, + testing up to 20000 covers some room for future expansion */ + for (int i=0;i<20000;i++) + { + if(i!=ERROR_SUCCESS) + cl_assert_(errnos_contains(err_win_to_posix(i)), + "all err_win_to_posix return values should be tested against mingw_strerror"); + /* ideally we'd test the same for winsock_error_to_errno, but it's static */ + } +#else + cl_skip(); +#endif +} From 0dc9d91859021554ef41a44d46763814e9aadeb9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 26 Nov 2025 17:51:41 +0100 Subject: [PATCH 086/229] t5563: verify that NTLM authentication works Although NTLM authentication is considered weak (extending even to NTLMv2, which purportedly allows brute-forcing reasonably complex 8-character passwords in a matter of days, given ample compute resources), it _is_ one of the authentication methods supported by libcurl. Note: The added test case *cannot* reuse the existing `custom_auth` facility. The reason is that that facility is backed by an NPH script ("No Parse Headers"), which does not allow handling the 3-phase NTLM authentication correctly (in my hands, the NPH script would not even be called upon the Type 3 message, a "200 OK" would be returned, but no headers, let alone the `git http-backend` output as payload). Having a separate NTLM authentication script makes the exact workings clearer and more readable, anyway. Co-authored-by: Matthew John Cheetham Signed-off-by: Johannes Schindelin --- t/lib-httpd.sh | 1 + t/lib-httpd/apache.conf | 8 ++++ t/lib-httpd/ntlm-handshake.sh | 38 +++++++++++++++++++ t/t5563-simple-http-auth.sh | 71 +++-------------------------------- 4 files changed, 53 insertions(+), 65 deletions(-) create mode 100755 t/lib-httpd/ntlm-handshake.sh diff --git a/t/lib-httpd.sh b/t/lib-httpd.sh index 554b0e44fa384f..bbcd4382b9eff8 100644 --- a/t/lib-httpd.sh +++ b/t/lib-httpd.sh @@ -181,6 +181,7 @@ prepare_httpd() { install_script apply-one-time-script.sh install_script nph-custom-auth.sh install_script http-429.sh + install_script ntlm-handshake.sh ln -s "$LIB_HTTPD_MODULE_PATH" "$HTTPD_ROOT_PATH/modules" diff --git a/t/lib-httpd/apache.conf b/t/lib-httpd/apache.conf index f78f9cc6427e70..a2350a69042378 100644 --- a/t/lib-httpd/apache.conf +++ b/t/lib-httpd/apache.conf @@ -156,6 +156,13 @@ SetEnv PERL_PATH ${PERL_PATH} CGIPassAuth on + + SetEnv GIT_EXEC_PATH ${GIT_EXEC_PATH} + SetEnv GIT_HTTP_EXPORT_ALL + + CGIPassAuth on + + ScriptAlias /smart/incomplete_length/git-upload-pack incomplete-length-upload-pack-v2-http.sh/ ScriptAlias /smart/incomplete_body/git-upload-pack incomplete-body-upload-pack-v2-http.sh/ ScriptAlias /smart/no_report/git-receive-pack error-no-report.sh/ @@ -167,6 +174,7 @@ ScriptAlias /error/ error.sh/ ScriptAliasMatch /one_time_script/(.*) apply-one-time-script.sh/$1 ScriptAliasMatch /http_429/(.*) http-429.sh/$1 ScriptAliasMatch /custom_auth/(.*) nph-custom-auth.sh/$1 +ScriptAliasMatch /ntlm_auth/(.*) ntlm-handshake.sh/$1 Options FollowSymlinks diff --git a/t/lib-httpd/ntlm-handshake.sh b/t/lib-httpd/ntlm-handshake.sh new file mode 100755 index 00000000000000..3cf1266e40f20a --- /dev/null +++ b/t/lib-httpd/ntlm-handshake.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +case "$HTTP_AUTHORIZATION" in +'') + # No Authorization header -> send NTLM challenge + echo "Status: 401 Unauthorized" + echo "WWW-Authenticate: NTLM" + echo + ;; +"NTLM TlRMTVNTUAAB"*) + # Type 1 -> respond with Type 2 challenge (hardcoded) + echo "Status: 401 Unauthorized" + # Base64-encoded version of the Type 2 challenge: + # signature: 'NTLMSSP\0' + # message_type: 2 + # target_name: 'NTLM-GIT-SERVER' + # flags: 0xa2898205 = + # NEGOTIATE_UNICODE, REQUEST_TARGET, NEGOTIATE_NT_ONLY, + # TARGET_TYPE_SERVER, TARGET_TYPE_SHARE, REQUEST_NON_NT_SESSION_KEY, + # NEGOTIATE_VERSION, NEGOTIATE_128, NEGOTIATE_56 + # challenge: 0xfa3dec518896295b + # context: '0000000000000000' + # target_info_present: true + # target_info_len: 128 + # version: '10.0 (build 19041)' + echo "WWW-Authenticate: NTLM TlRMTVNTUAACAAAAHgAeADgAAAAFgomi+j3sUYiWKVsAAAAAAAAAAIAAgABWAAAACgBhSgAAAA9OAFQATABNAC0ARwBJAFQALQBTAEUAUgBWAEUAUgACABIAVwBPAFIASwBHAFIATwBVAFAAAQAeAE4AVABMAE0ALQBHAEkAVAAtAFMARQBSAFYARQBSAAQAEgBXAE8AUgBLAEcAUgBPAFUAUAADAB4ATgBUAEwATQAtAEcASQBUAC0AUwBFAFIAVgBFAFIABwAIAACfOcZKYNwBAAAAAA==" + echo + ;; +"NTLM TlRMTVNTUAAD"*) + # Type 3 -> accept without validation + exec "$GIT_EXEC_PATH"/git-http-backend + ;; +*) + echo "Status: 500 Unrecognized" + echo + echo "Unhandled auth: '$HTTP_AUTHORIZATION'" + ;; +esac diff --git a/t/t5563-simple-http-auth.sh b/t/t5563-simple-http-auth.sh index 349ae4ab39cb0f..2082e553ae975a 100755 --- a/t/t5563-simple-http-auth.sh +++ b/t/t5563-simple-http-auth.sh @@ -764,78 +764,19 @@ test_expect_success 'access using three-legged auth' ' EOF ' -test_lazy_prereq SPNEGO 'curl --version | grep -qi "SPNEGO\|GSS-API\|Kerberos\|negotiate"' +test_lazy_prereq NTLM 'curl --version | grep -q NTLM' -test_expect_success SPNEGO 'http.emptyAuth=auto attempts Negotiate before credential_fill' ' +test_expect_success NTLM 'access using NTLM auth' ' test_when_finished "per_test_cleanup" && set_credential_reply get <<-EOF && - username=alice - password=secret-passwd - EOF - - # Basic base64(alice:secret-passwd) - cat >"$HTTPD_ROOT_PATH/custom-auth.valid" <<-EOF && - id=1 creds=Basic YWxpY2U6c2VjcmV0LXBhc3N3ZA== - EOF - - cat >"$HTTPD_ROOT_PATH/custom-auth.challenge" <<-EOF && - id=1 status=200 - id=default response=WWW-Authenticate: Negotiate - id=default response=WWW-Authenticate: Basic realm="example.com" - EOF - - test_config_global credential.helper test-helper && - GIT_TRACE_CURL="$TRASH_DIRECTORY/trace-auto" \ - git -c http.emptyAuth=auto \ - ls-remote "$HTTPD_URL/custom_auth/repo.git" && - - # In auto mode with a Negotiate+Basic server, there should be - # three 401 responses: (1) initial no-auth request, (2) empty-auth - # retry where Negotiate fails (no Kerberos ticket), (3) libcurl - # internal Negotiate retry. The fourth attempt uses Basic - # credentials from credential_fill and succeeds. - grep "HTTP/[0-9.]* 401" "$TRASH_DIRECTORY/trace-auto" >actual_401s && - test_line_count = 3 actual_401s && - - expect_credential_query get <<-EOF - capability[]=authtype - capability[]=state - protocol=http - host=$HTTPD_DEST - wwwauth[]=Negotiate - wwwauth[]=Basic realm="example.com" - EOF -' - -test_expect_success SPNEGO 'http.emptyAuth=false skips Negotiate' ' - test_when_finished "per_test_cleanup" && - - set_credential_reply get <<-EOF && - username=alice - password=secret-passwd - EOF - - # Basic base64(alice:secret-passwd) - cat >"$HTTPD_ROOT_PATH/custom-auth.valid" <<-EOF && - id=1 creds=Basic YWxpY2U6c2VjcmV0LXBhc3N3ZA== - EOF - - cat >"$HTTPD_ROOT_PATH/custom-auth.challenge" <<-EOF && - id=1 status=200 - id=default response=WWW-Authenticate: Negotiate - id=default response=WWW-Authenticate: Basic realm="example.com" + username=user + password=pwd EOF test_config_global credential.helper test-helper && - GIT_TRACE_CURL="$TRASH_DIRECTORY/trace-false" \ - git -c http.emptyAuth=false \ - ls-remote "$HTTPD_URL/custom_auth/repo.git" && - - # With emptyAuth=false, Negotiate is stripped immediately and - # credential_fill is called right away. Only one 401 response. - grep "HTTP/[0-9.]* 401" "$TRASH_DIRECTORY/trace-false" >actual_401s && - test_line_count = 1 actual_401s + GIT_TRACE_CURL=1 \ + git ls-remote "$HTTPD_URL/ntlm_auth/repo.git" ' test_done From 6673170db173a23a22a556ecd389bb1ef38b2aa5 Mon Sep 17 00:00:00 2001 From: JiSeop Moon Date: Mon, 23 Apr 2018 22:30:18 +0900 Subject: [PATCH 087/229] mingw: introduce code to detect whether we're inside a Windows container This will come in handy in the next commit. Signed-off-by: JiSeop Moon Signed-off-by: Johannes Schindelin --- compat/mingw.c | 32 ++++++++++++++++++++++++++++++++ compat/mingw.h | 5 +++++ 2 files changed, 37 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..bc4d54db58df0e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3853,3 +3853,35 @@ int mingw_have_unix_sockets(void) return ret; } #endif + +/* + * Based on https://stackoverflow.com/questions/43002803 + * + * [HKLM\SYSTEM\CurrentControlSet\Services\cexecsvc] + * "DisplayName"="@%systemroot%\\system32\\cexecsvc.exe,-100" + * "ErrorControl"=dword:00000001 + * "ImagePath"=hex(2):25,00,73,00,79,00,73,00,74,00,65,00,6d,00,72,00,6f,00, + * 6f,00,74,00,25,00,5c,00,73,00,79,00,73,00,74,00,65,00,6d,00,33,00,32,00, + * 5c,00,63,00,65,00,78,00,65,00,63,00,73,00,76,00,63,00,2e,00,65,00,78,00, + * 65,00,00,00 + * "Start"=dword:00000002 + * "Type"=dword:00000010 + * "Description"="@%systemroot%\\system32\\cexecsvc.exe,-101" + * "ObjectName"="LocalSystem" + * "ServiceSidType"=dword:00000001 + */ +int is_inside_windows_container(void) +{ + static int inside_container = -1; /* -1 uninitialized */ + const char *key = "SYSTEM\\CurrentControlSet\\Services\\cexecsvc"; + HKEY handle = NULL; + + if (inside_container != -1) + return inside_container; + + inside_container = ERROR_SUCCESS == + RegOpenKeyExA(HKEY_LOCAL_MACHINE, key, 0, KEY_READ, &handle); + RegCloseKey(handle); + + return inside_container; +} diff --git a/compat/mingw.h b/compat/mingw.h index df02aeb632d8c3..745cc657b76b98 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -216,3 +216,8 @@ int mingw_have_unix_sockets(void); #undef have_unix_sockets #define have_unix_sockets mingw_have_unix_sockets #endif + +/* + * Check current process is inside Windows Container. + */ +int is_inside_windows_container(void); From fc051d699e6a3a60f670e515a5654130a7d8bd8a Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 26 Oct 2018 11:13:45 +0200 Subject: [PATCH 088/229] Win32: symlink: move phantom symlink creation to a separate function Signed-off-by: Bert Belder --- compat/mingw.c | 91 +++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 42 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..b3af992def50bf 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -449,6 +449,54 @@ static void process_phantom_symlinks(void) LeaveCriticalSection(&phantom_symlinks_cs); } +static int create_phantom_symlink(wchar_t *wtarget, wchar_t *wlink) +{ + int len; + + /* create file symlink */ + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* convert to directory symlink if target exists */ + switch (process_phantom_symlink(wtarget, wlink)) { + case PHANTOM_SYMLINK_RETRY: { + /* if target doesn't exist, add to phantom symlinks list */ + wchar_t wfullpath[MAX_LONG_PATH]; + struct phantom_symlink_info *psi; + + /* convert to absolute path to be independent of cwd */ + len = GetFullPathNameW(wlink, MAX_LONG_PATH, wfullpath, NULL); + if (!len || len >= MAX_LONG_PATH) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* over-allocate and fill phantom_symlink_info structure */ + psi = xmalloc(sizeof(struct phantom_symlink_info) + + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); + psi->wlink = (wchar_t *)(psi + 1); + wcscpy(psi->wlink, wfullpath); + psi->wtarget = psi->wlink + len + 1; + wcscpy(psi->wtarget, wtarget); + + EnterCriticalSection(&phantom_symlinks_cs); + psi->next = phantom_symlinks; + phantom_symlinks = psi; + LeaveCriticalSection(&phantom_symlinks_cs); + break; + } + case PHANTOM_SYMLINK_DIRECTORY: + /* if we created a dir symlink, process other phantom symlinks */ + process_phantom_symlinks(); + break; + default: + break; + } + return 0; +} + /* Normalizes NT paths as returned by some low-level APIs. */ static wchar_t *normalize_ntpath(wchar_t *wbuf) { @@ -2952,48 +3000,7 @@ int symlink(const char *target, const char *link) if (wtarget[len] == '/') wtarget[len] = '\\'; - /* create file symlink */ - if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) { - errno = err_win_to_posix(GetLastError()); - return -1; - } - - /* convert to directory symlink if target exists */ - switch (process_phantom_symlink(wtarget, wlink)) { - case PHANTOM_SYMLINK_RETRY: { - /* if target doesn't exist, add to phantom symlinks list */ - wchar_t wfullpath[MAX_PATH]; - struct phantom_symlink_info *psi; - - /* convert to absolute path to be independent of cwd */ - len = GetFullPathNameW(wlink, MAX_PATH, wfullpath, NULL); - if (!len || len >= MAX_PATH) { - errno = err_win_to_posix(GetLastError()); - return -1; - } - - /* over-allocate and fill phantom_symlink_info structure */ - psi = xmalloc(sizeof(struct phantom_symlink_info) - + sizeof(wchar_t) * (len + wcslen(wtarget) + 2)); - psi->wlink = (wchar_t *)(psi + 1); - wcscpy(psi->wlink, wfullpath); - psi->wtarget = psi->wlink + len + 1; - wcscpy(psi->wtarget, wtarget); - - EnterCriticalSection(&phantom_symlinks_cs); - psi->next = phantom_symlinks; - phantom_symlinks = psi; - LeaveCriticalSection(&phantom_symlinks_cs); - break; - } - case PHANTOM_SYMLINK_DIRECTORY: - /* if we created a dir symlink, process other phantom symlinks */ - process_phantom_symlinks(); - break; - default: - break; - } - return 0; + return create_phantom_symlink(wtarget, wlink); } int readlink(const char *path, char *buf, size_t bufsiz) From 3beeee92c2856aaa8b1995e560ec63435862a9ea Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 17:33:36 +0200 Subject: [PATCH 089/229] xdiff-interface: widen buffer_is_binary() size parameter to size_t Prep for the widenings of its callers, where size-receiving locals will become size_t (combine-diff's result_size in the immediately following commit, struct diff_filespec.size in a later topic). Body caps the parameter at 8000 anyway, so the type change is mechanical. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- xdiff-interface.c | 2 +- xdiff-interface.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/xdiff-interface.c b/xdiff-interface.c index 233b9c4754de46..142dbc6eba91a0 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -215,7 +215,7 @@ void read_mmblob(mmfile_t *ptr, struct object_database *odb, } #define FIRST_FEW_BYTES 8000 -int buffer_is_binary(const char *ptr, unsigned long size) +int buffer_is_binary(const char *ptr, size_t size) { if (FIRST_FEW_BYTES < size) size = FIRST_FEW_BYTES; diff --git a/xdiff-interface.h b/xdiff-interface.h index c4f5aa92fdf1ef..27e5e6bf679229 100644 --- a/xdiff-interface.h +++ b/xdiff-interface.h @@ -71,7 +71,7 @@ int diff_emit_line_ranges(mmfile_t *mf1, mmfile_t *mf2, int read_mmfile(mmfile_t *ptr, const char *filename); void read_mmblob(mmfile_t *ptr, struct object_database *odb, const struct object_id *oid); -int buffer_is_binary(const char *ptr, unsigned long size); +int buffer_is_binary(const char *ptr, size_t size); void xdiff_set_find_func(xdemitconf_t *xecfg, const char *line, int cflags); void xdiff_clear_find_func(xdemitconf_t *xecfg); From 57a3d5fd079f93c4403fa3009c183a29affc4f8b Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Sun, 1 Sep 2024 22:35:40 -0400 Subject: [PATCH 090/229] survey: add report of "largest" paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since we are already walking our reachable objects using the path-walk API, let's now collect lists of the paths that contribute most to different metrics. Specifically, we care about * Number of versions. * Total size on disk. * Total inflated size (no delta or zlib compression). This information can be critical to discovering which parts of the repository are causing the most growth, especially on-disk size. Different packing strategies might help compress data more efficiently, but the toal inflated size is a representation of the raw size of all snapshots of those paths. Even when stored efficiently on disk, that size represents how much information must be processed to complete a command such as 'git blame'. The exact disk size seems to be not quite robust enough for testing, as could be seen by the `linux-musl-meson` job consistently failing, possibly because of zlib-ng deflates differently: t8100.4(git survey (default)) was failing with a symptom like this: TOTAL OBJECT SIZES BY TYPE =============================================== Object Type | Count | Disk Size | Inflated Size ------------+-------+-----------+-------------- - Commits | 10 | 1523 | 2153 + Commits | 10 | 1528 | 2153 Trees | 10 | 495 | 1706 Blobs | 10 | 191 | 101 - Tags | 4 | 510 | 528 + Tags | 4 | 547 | 528 This means: the disk size is unlikely something we can verify robustly. Since zlib-ng seems to increase the disk size of the tags from 528 to 547, we cannot even assume that the disk size is always smaller than the inflated size. We will most likely want to either skip verifying the disk size altogether, or go for some kind of fuzzy matching, say, by replacing `s/ 1[45][0-9][0-9] / ~1.5k /` and `s/ [45][0-9][0-9] / ~½k /` or something like that. Signed-off-by: Derrick Stolee Signed-off-by: Johannes Schindelin --- builtin/survey.c | 79 ++++++++++++++++++++++++++++++++++++++----- t/t8100-git-survey.sh | 12 ++++++- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/builtin/survey.c b/builtin/survey.c index c513b653f9604a..a0a3966295340f 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -75,7 +75,6 @@ struct survey_report_object_size_summary { typedef int (*survey_top_cmp)(void *v1, void *v2); -MAYBE_UNUSED static int cmp_by_nr(void *v1, void *v2) { struct survey_report_object_size_summary *s1 = v1; @@ -88,7 +87,6 @@ static int cmp_by_nr(void *v1, void *v2) return 0; } -MAYBE_UNUSED static int cmp_by_disk_size(void *v1, void *v2) { struct survey_report_object_size_summary *s1 = v1; @@ -101,7 +99,6 @@ static int cmp_by_disk_size(void *v1, void *v2) return 0; } -MAYBE_UNUSED static int cmp_by_inflated_size(void *v1, void *v2) { struct survey_report_object_size_summary *s1 = v1; @@ -132,7 +129,6 @@ struct survey_report_top_table { void *data; }; -MAYBE_UNUSED static void init_top_sizes(struct survey_report_top_table *top, size_t limit, const char *name, survey_top_cmp cmp) @@ -158,7 +154,6 @@ static void clear_top_sizes(struct survey_report_top_table *top) free(sz_array); } -MAYBE_UNUSED static void maybe_insert_into_top_size(struct survey_report_top_table *top, struct survey_report_object_size_summary *summary) { @@ -195,6 +190,10 @@ struct survey_report { struct survey_report_object_summary reachable_objects; struct survey_report_object_size_summary *by_type; + + struct survey_report_top_table *top_paths_by_count; + struct survey_report_top_table *top_paths_by_disk; + struct survey_report_top_table *top_paths_by_inflate; }; #define REPORT_TYPE_COMMIT 0 @@ -446,6 +445,13 @@ static void survey_report_object_sizes(const char *title, clear_table(&table); } +static void survey_report_plaintext_sorted_size( + struct survey_report_top_table *top) +{ + survey_report_object_sizes(top->name, _("Path"), + top->data, top->nr); +} + static void survey_report_plaintext(struct survey_context *ctx) { printf("GIT SURVEY for \"%s\"\n", ctx->repo->worktree); @@ -456,6 +462,21 @@ static void survey_report_plaintext(struct survey_context *ctx) _("Object Type"), ctx->report.by_type, REPORT_TYPE_COUNT); + + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_count[REPORT_TYPE_TREE]); + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_count[REPORT_TYPE_BLOB]); + + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_disk[REPORT_TYPE_TREE]); + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_disk[REPORT_TYPE_BLOB]); + + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE]); + survey_report_plaintext_sorted_size( + &ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB]); } /* @@ -698,7 +719,8 @@ static void increment_totals(struct survey_context *ctx, static void increment_object_totals(struct survey_context *ctx, struct oid_array *oids, - enum object_type type) + enum object_type type, + const char *path) { struct survey_report_object_size_summary *total; struct survey_report_object_size_summary summary = { 0 }; @@ -730,9 +752,30 @@ static void increment_object_totals(struct survey_context *ctx, total->disk_size += summary.disk_size; total->inflated_size += summary.inflated_size; total->num_missing += summary.num_missing; + + if (type == OBJ_TREE || type == OBJ_BLOB) { + int index = type == OBJ_TREE ? + REPORT_TYPE_TREE : REPORT_TYPE_BLOB; + struct survey_report_top_table *top; + + /* + * Temporarily store (const char *) here, but it will + * be duped if inserted and will not be freed. + */ + summary.label = (char *)path; + + top = ctx->report.top_paths_by_count; + maybe_insert_into_top_size(&top[index], &summary); + + top = ctx->report.top_paths_by_disk; + maybe_insert_into_top_size(&top[index], &summary); + + top = ctx->report.top_paths_by_inflate; + maybe_insert_into_top_size(&top[index], &summary); + } } -static int survey_objects_path_walk_fn(const char *path UNUSED, +static int survey_objects_path_walk_fn(const char *path, struct oid_array *oids, enum object_type type, void *data) @@ -741,7 +784,7 @@ static int survey_objects_path_walk_fn(const char *path UNUSED, increment_object_counts(&ctx->report.reachable_objects, type, oids->nr); - increment_object_totals(ctx, oids, type); + increment_object_totals(ctx, oids, type, path); ctx->progress_nr += oids->nr; display_progress(ctx->progress, ctx->progress_nr); @@ -751,11 +794,31 @@ static int survey_objects_path_walk_fn(const char *path UNUSED, static void initialize_report(struct survey_context *ctx) { + const int top_limit = 100; + CALLOC_ARRAY(ctx->report.by_type, REPORT_TYPE_COUNT); ctx->report.by_type[REPORT_TYPE_COMMIT].label = xstrdup(_("Commits")); ctx->report.by_type[REPORT_TYPE_TREE].label = xstrdup(_("Trees")); ctx->report.by_type[REPORT_TYPE_BLOB].label = xstrdup(_("Blobs")); ctx->report.by_type[REPORT_TYPE_TAG].label = xstrdup(_("Tags")); + + CALLOC_ARRAY(ctx->report.top_paths_by_count, REPORT_TYPE_COUNT); + init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_TREE], + top_limit, _("TOP DIRECTORIES BY COUNT"), cmp_by_nr); + init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_BLOB], + top_limit, _("TOP FILES BY COUNT"), cmp_by_nr); + + CALLOC_ARRAY(ctx->report.top_paths_by_disk, REPORT_TYPE_COUNT); + init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_TREE], + top_limit, _("TOP DIRECTORIES BY DISK SIZE"), cmp_by_disk_size); + init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_BLOB], + top_limit, _("TOP FILES BY DISK SIZE"), cmp_by_disk_size); + + CALLOC_ARRAY(ctx->report.top_paths_by_inflate, REPORT_TYPE_COUNT); + init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE], + top_limit, _("TOP DIRECTORIES BY INFLATED SIZE"), cmp_by_inflated_size); + init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB], + top_limit, _("TOP FILES BY INFLATED SIZE"), cmp_by_inflated_size); } static void survey_phase_objects(struct survey_context *ctx) diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh index b63ba3a1c8e310..703860d22ff4c3 100755 --- a/t/t8100-git-survey.sh +++ b/t/t8100-git-survey.sh @@ -92,7 +92,17 @@ test_expect_success 'git survey (default)' ' EOF approximate_sizes out >out-edited && - test_cmp expect out-edited + lines=$(wc -l out-trimmed && + test_cmp expect out-trimmed && + + for type in "DIRECTORIES" "FILES" + do + for metric in "COUNT" "DISK SIZE" "INFLATED SIZE" + do + grep "TOP $type BY $metric" out || return 1 + done || return 1 + done ' test_done From 9d243399dfb777d18bdda80c86d3c2e21af01055 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 6 Jun 2026 00:23:37 +0200 Subject: [PATCH 091/229] repo: read the `--top` default from `repo.structure.top` `git survey` exposes its `--top` default via `survey.top` so that a site or per-repository operator can switch the detail tables on once and have every subsequent invocation include them. Mirror that ergonomics for `git repo structure` so that, as `git survey`'s functionality is folded into `git repo structure`, the configuration side of the migration story stays equivalent. Add a small `git_config_int` callback bound to `repo.structure.top` and invoke it before `parse_options()`, so a `--top=` on the command line cleanly overrides the configured default (including `--top=0` to opt out of the detail tables when configuration enables them). Reject negative configured values with the same wording as the command-line guard, since `git_config_int()` happily returns negative integers. Document the new variable in a fresh `Documentation/config/repo.adoc` and wire it into the alphabetical includes in `Documentation/config.adoc` between `repack.adoc` and `rerere.adoc`. Cover the precedence behaviour with a t1901 test: a configured value enables the tables by default, and a command-line `--top=0` suppresses them again. Note that the reported paths respect the `core.quotePath` setting. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- Documentation/config.adoc | 2 ++ Documentation/config/repo.adoc | 11 +++++++++++ builtin/repo.c | 19 +++++++++++++++++++ t/t1901-repo-structure.sh | 17 +++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 Documentation/config/repo.adoc diff --git a/Documentation/config.adoc b/Documentation/config.adoc index 758c31d38e813c..6da3bb88cdddad 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -572,6 +572,8 @@ include::config/repack.adoc[] include::config/replay.adoc[] +include::config/repo.adoc[] + include::config/rerere.adoc[] include::config/revert.adoc[] diff --git a/Documentation/config/repo.adoc b/Documentation/config/repo.adoc new file mode 100644 index 00000000000000..7f8cd632965aab --- /dev/null +++ b/Documentation/config/repo.adoc @@ -0,0 +1,11 @@ +repo.structure.*:: + These variables adjust the default behavior of the + `git repo structure` command. ++ +-- + top:: + This integer value implies `--top=`, specifying the + number of largest paths to report in each detail table. + Must be non-negative; defaults to `0`, which disables the + detail tables. +-- diff --git a/builtin/repo.c b/builtin/repo.c index 30c1685a1e2e65..2440da9cdaf589 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -2,6 +2,7 @@ #include "builtin.h" #include "commit.h" +#include "config.h" #include "environment.h" #include "hash.h" #include "hex.h" @@ -1340,6 +1341,22 @@ static void structure_count_objects(struct object_stats *stats, stop_progress(&data.progress); } +static int repo_structure_config_cb(const char *var, const char *value, + const struct config_context *cctx, + void *cb) +{ + int *top_nr = cb; + + if (!strcmp(var, "repo.structure.top")) { + *top_nr = git_config_int(var, value, cctx->kvi); + if (*top_nr < 0) + die(_("repo.structure.top must be non-negative")); + return 0; + } + + return git_default_config(var, value, cctx, NULL); +} + static int cmd_repo_structure(int argc, const char **argv, const char *prefix, struct repository *repo) { @@ -1371,6 +1388,8 @@ static int cmd_repo_structure(int argc, const char **argv, const char *prefix, OPT_END() }; + repo_config(repo, repo_structure_config_cb, &top_nr); + argc = parse_options(argc, argv, prefix, options, repo_structure_usage, 0); if (argc) usage(_("too many arguments")); diff --git a/t/t1901-repo-structure.sh b/t/t1901-repo-structure.sh index baafd7fbe756ec..c2c198c43e9922 100755 --- a/t/t1901-repo-structure.sh +++ b/t/t1901-repo-structure.sh @@ -313,6 +313,23 @@ test_expect_success '--top rejects negative values' ' test_grep "must be non-negative" err ' +test_expect_success 'repo.structure.top supplies the default for --top' ' + test_when_finished "rm -rf repo" && + git init repo && + ( + cd repo && + test_commit foo && + + git -c repo.structure.top=2 \ + repo structure --format=lines >with-config && + test_grep "^objects.blobs.top.by_count.1.path=" with-config && + + git -c repo.structure.top=2 \ + repo structure --format=lines --top=0 >cli-override && + test_grep ! "\.top\." cli-override + ) +' + test_expect_success 'git repo structure -h shows only repo structure usage' ' git repo structure -h >actual && test_grep "git repo structure" actual && From ac059d5986cc3d2acb3a973706d864f9cb9fb600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20A=C3=9Fhauer?= Date: Sun, 22 Dec 2024 17:43:45 +0100 Subject: [PATCH 092/229] compat/mingw: drop outdated comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This comment has been true for the longest time; The combination of the two preceding commits made it incorrect, so let's drop that comment. Signed-off-by: Matthias Aßhauer Signed-off-by: Johannes Schindelin --- compat/mingw.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index d0416f8d8e2664..c4c57cb12022ea 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2666,15 +2666,6 @@ int mingw_socket(int domain, int type, int protocol) ensure_socket_initialization(); s = WSASocket(domain, type, protocol, NULL, 0, 0); if (s == INVALID_SOCKET) { - /* - * WSAGetLastError() values are regular BSD error codes - * biased by WSABASEERR. - * However, strerror() does not know about networking - * specific errors, which are values beginning at 38 or so. - * Therefore, we choose to leave the biased error code - * in errno so that _if_ someone looks up the code somewhere, - * then it is at least the number that are usually listed. - */ set_wsa_errno(); return -1; } From 148f2e9db10cadfd297e37082f871ca2ba500ecd Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 26 Nov 2025 18:47:19 +0100 Subject: [PATCH 093/229] http: disallow NTLM authentication by default NTLM authentication is relatively weak. This is the case even with the default setting of modern Windows versions, where NTLMv1 and LanManager are disabled and only NTLMv2 is enabled: NTLMv2 hashes of even reasonably complex 8-character passwords can be broken in a matter of days, given enough compute resources. Even worse: On Windows, NTLM authentication uses Security Support Provider Interface ("SSPI"), which provides the credentials without requiring the user to type them in. Which means that an attacker could talk an unsuspecting user into cloning from a server that is under the attacker's control and extracts the user's NTLMv2 hash without their knowledge. For that reason, let's disallow NTLM authentication by default. NTLM authentication is quite simple to set up, though, and therefore there are still some on-prem Azure DevOps setups out there whose users and/or automation rely on this type of authentication. To give them an escape hatch, introduce the `http..allowNTLMAuth` config setting that can be set to `true` to opt back into using NTLM for a specific remote repository. Signed-off-by: Johannes Schindelin --- Documentation/config/http.adoc | 5 +++++ http.c | 20 ++++++++++++++++---- t/t5563-simple-http-auth.sh | 6 ++++-- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/Documentation/config/http.adoc b/Documentation/config/http.adoc index b54f62796921c1..5c5e0fd966e363 100644 --- a/Documentation/config/http.adoc +++ b/Documentation/config/http.adoc @@ -256,6 +256,11 @@ http.sslKeyType:: See also libcurl `CURLOPT_SSLKEYTYPE`. Can be overridden by the `GIT_SSL_KEY_TYPE` environment variable. +http.allowNTLMAuth:: + Whether or not to allow NTLM authentication. While very convenient to set + up, and therefore still used in many on-prem scenarios, NTLM is a weak + authentication method and therefore deprecated. Defaults to "false". + http.schannelCheckRevoke:: Used to enforce or disable certificate revocation checks in cURL when http.sslBackend is set to "schannel". Defaults to `true` if diff --git a/http.c b/http.c index 334f33e3233dbb..4844de1d3eed7a 100644 --- a/http.c +++ b/http.c @@ -132,7 +132,8 @@ enum http_follow_config http_follow_config = HTTP_FOLLOW_INITIAL; static struct credential cert_auth = CREDENTIAL_INIT; static int ssl_cert_password_required; -static unsigned long http_auth_methods = CURLAUTH_ANY; +static unsigned long http_auth_any = CURLAUTH_ANY & ~CURLAUTH_NTLM; +static unsigned long http_auth_methods; static int http_auth_methods_restricted; /* Modes for which empty_auth cannot actually help us. */ static unsigned long empty_auth_useless = @@ -435,6 +436,15 @@ static int http_options(const char *var, const char *value, return 0; } + if (!strcmp("http.allowntlmauth", var)) { + if (git_config_bool(var, value)) { + http_auth_any |= CURLAUTH_NTLM; + } else { + http_auth_any &= ~CURLAUTH_NTLM; + } + return 0; + } + if (!strcmp("http.schannelcheckrevoke", var)) { http_schannel_check_revoke = git_config_bool(var, value); return 0; @@ -731,11 +741,11 @@ static void init_curl_proxy_auth(CURL *result) if (i == ARRAY_SIZE(proxy_authmethods)) { warning("unsupported proxy authentication method %s: using anyauth", http_proxy_authmethod); - curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + curl_easy_setopt(result, CURLOPT_PROXYAUTH, http_auth_any); } } else - curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY); + curl_easy_setopt(result, CURLOPT_PROXYAUTH, http_auth_any); } static int has_cert_password(void) @@ -1156,7 +1166,7 @@ static CURL *get_curl_handle(void) } curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); - curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY); + curl_easy_setopt(result, CURLOPT_HTTPAUTH, http_auth_any); #ifdef CURLGSSAPI_DELEGATION_FLAG if (curl_deleg) { @@ -1524,6 +1534,8 @@ void http_init(struct remote *remote, const char *url, int proactive_auth) set_long_from_env(&http_max_retries, "GIT_HTTP_MAX_RETRIES"); set_long_from_env(&http_max_retry_time, "GIT_HTTP_MAX_RETRY_TIME"); + http_auth_methods = http_auth_any; + curl_default = get_curl_handle(); } diff --git a/t/t5563-simple-http-auth.sh b/t/t5563-simple-http-auth.sh index 2082e553ae975a..965c2fb71637c8 100755 --- a/t/t5563-simple-http-auth.sh +++ b/t/t5563-simple-http-auth.sh @@ -775,8 +775,10 @@ test_expect_success NTLM 'access using NTLM auth' ' EOF test_config_global credential.helper test-helper && - GIT_TRACE_CURL=1 \ - git ls-remote "$HTTPD_URL/ntlm_auth/repo.git" + test_must_fail env GIT_TRACE_CURL=1 git \ + ls-remote "$HTTPD_URL/ntlm_auth/repo.git" && + GIT_TRACE_CURL=1 git -c http.$HTTPD_URL.allowNTLMAuth=true \ + ls-remote "$HTTPD_URL/ntlm_auth/repo.git" ' test_done From 0c158a3248195f9daa559bc96f83e7f5f877a2d4 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Thu, 19 Mar 2015 16:33:44 +0100 Subject: [PATCH 094/229] mingw: Support `git_terminal_prompt` with more terminals The `git_terminal_prompt()` function expects the terminal window to be attached to a Win32 Console. However, this is not the case with terminal windows other than `cmd.exe`'s, e.g. with MSys2's own `mintty`. Non-cmd terminals such as `mintty` still have to have a Win32 Console to be proper console programs, but have to hide the Win32 Console to be able to provide more flexibility (such as being resizeable not only vertically but also horizontally). By writing to that Win32 Console, `git_terminal_prompt()` manages only to send the prompt to nowhere and to wait for input from a Console to which the user has no access. This commit introduces a function specifically to support `mintty` -- or other terminals that are compatible with MSys2's `/dev/tty` emulation. We use the `TERM` environment variable as an indicator for that: if the value starts with "xterm" (such as `mintty`'s "xterm_256color"), we prefer to let `xterm_prompt()` handle the user interaction. The most prominent user of `git_terminal_prompt()` is certainly `git-remote-https.exe`. It is an interesting use case because both `stdin` and `stdout` are redirected when Git calls said executable, yet it still wants to access the terminal. When running inside a `mintty`, the terminal is not accessible to the `git-remote-https.exe` program, though, because it is a MinGW program and the `mintty` terminal is not backed by a Win32 console. To solve that problem, we simply call out to the shell -- which is an *MSys2* program and can therefore access `/dev/tty`. Helped-by: nalla Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/terminal.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/compat/terminal.c b/compat/terminal.c index 584f27bf7e1078..cdcde283644e41 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -418,6 +418,54 @@ static int getchar_with_timeout(int timeout) return getchar(); } +static char *shell_prompt(const char *prompt, int echo) +{ + const char *read_input[] = { + /* Note: call 'bash' explicitly, as 'read -s' is bash-specific */ + "bash", "-c", echo ? + "cat >/dev/tty && read -r line /dev/tty && read -r -s line /dev/tty", + NULL + }; + struct child_process child = CHILD_PROCESS_INIT; + static struct strbuf buffer = STRBUF_INIT; + int prompt_len = strlen(prompt), len = -1, code; + + strvec_pushv(&child.args, read_input); + child.in = -1; + child.out = -1; + + if (start_command(&child)) + return NULL; + + if (write_in_full(child.in, prompt, prompt_len) != prompt_len) { + error("could not write to prompt script"); + close(child.in); + goto ret; + } + close(child.in); + + strbuf_reset(&buffer); + len = strbuf_read(&buffer, child.out, 1024); + if (len < 0) { + error("could not read from prompt script"); + goto ret; + } + + strbuf_strip_suffix(&buffer, "\n"); + strbuf_strip_suffix(&buffer, "\r"); + +ret: + close(child.out); + code = finish_command(&child); + if (code) { + error("failed to execute prompt script (exit code %d)", code); + return NULL; + } + + return len < 0 ? NULL : buffer.buf; +} + #endif #ifndef FORCE_TEXT @@ -429,6 +477,12 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; +#ifdef GIT_WINDOWS_NATIVE + const char *term = getenv("TERM"); + + if (term && starts_with(term, "xterm")) + return shell_prompt(prompt, echo); +#endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); if (!input_fh) From 9eab9aa85e1e0ad32ddbd3fde7e8c8dfb9f815a7 Mon Sep 17 00:00:00 2001 From: JiSeop Moon Date: Mon, 23 Apr 2018 22:31:42 +0200 Subject: [PATCH 095/229] mingw: when running in a Windows container, try to rename() harder It is a known issue that a rename() can fail with an "Access denied" error at times, when copying followed by deleting the original file works. Let's just fall back to that behavior. Signed-off-by: JiSeop Moon Signed-off-by: Johannes Schindelin --- compat/mingw.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index bc4d54db58df0e..e6e85e4be4028d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2619,6 +2619,13 @@ int mingw_rename(const char *pold, const char *pnew) gle = GetLastError(); } + if (gle == ERROR_ACCESS_DENIED && is_inside_windows_container()) { + /* Fall back to copy to destination & remove source */ + if (CopyFileW(wpold, wpnew, FALSE) && !mingw_unlink(pold, 1)) + return 0; + gle = GetLastError(); + } + /* revert file attributes on failure */ if (attrs != INVALID_FILE_ATTRIBUTES) SetFileAttributesW(wpnew, attrs); From 983a92d70b929aada302068aa24a586c52911fe4 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 11 Feb 2019 14:19:18 +0100 Subject: [PATCH 096/229] Introduce helper to create symlinks that knows about index_state On Windows, symbolic links actually have a type depending on the target: it can be a file or a directory. In certain circumstances, this poses problems, e.g. when a symbolic link is supposed to point into a submodule that is not checked out, so there is no way for Git to auto-detect the type. To help with that, we will add support over the course of the next commits to specify that symlink type via the Git attributes. This requires an index_state, though, something that Git for Windows' `symlink()` replacement cannot know about because the function signature is defined by the POSIX standard and not ours to change. So let's introduce a helper function to create symbolic links that *does* know about the index_state. Signed-off-by: Johannes Schindelin --- apply.c | 2 +- builtin/difftool.c | 2 +- compat/mingw-posix.h | 4 +++- compat/mingw.c | 2 +- entry.c | 2 +- git-compat-util.h | 10 ++++++++++ refs/files-backend.c | 2 +- setup.c | 4 ++-- 8 files changed, 20 insertions(+), 8 deletions(-) diff --git a/apply.c b/apply.c index ca8d905c3d5025..a1dd02c9aefc9e 100644 --- a/apply.c +++ b/apply.c @@ -4529,7 +4529,7 @@ static int try_create_file(struct apply_state *state, const char *path, /* Although buf:size is counted string, it also is NUL * terminated. */ - return !!symlink(buf, path); + return !!create_symlink(state && state->repo ? state->repo->index : NULL, buf, path); fd = open(path, O_CREAT | O_EXCL | O_WRONLY, (mode & 0100) ? 0777 : 0666); if (fd < 0) diff --git a/builtin/difftool.c b/builtin/difftool.c index bc7b2ea44396bb..ba33402308577b 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -544,7 +544,7 @@ static int run_dir_diff(struct repository *repo, } add_path(&wtdir, wtdir_len, dst_path); if (dt_options->symlinks) { - if (symlink(wtdir.buf, rdir.buf)) { + if (create_symlink(lstate.istate, wtdir.buf, rdir.buf)) { ret = error_errno("could not symlink '%s' to '%s'", wtdir.buf, rdir.buf); goto finish; } diff --git a/compat/mingw-posix.h b/compat/mingw-posix.h index 2d989fd762474e..2c96303a83e720 100644 --- a/compat/mingw-posix.h +++ b/compat/mingw-posix.h @@ -193,8 +193,10 @@ int setitimer(int type, struct itimerval *in, struct itimerval *out); int sigaction(int sig, struct sigaction *in, struct sigaction *out); int link(const char *oldpath, const char *newpath); int uname(struct utsname *buf); -int symlink(const char *target, const char *link); int readlink(const char *path, char *buf, size_t bufsiz); +struct index_state; +int mingw_create_symlink(struct index_state *index, const char *target, const char *link); +#define create_symlink mingw_create_symlink /* * replacements of existing functions diff --git a/compat/mingw.c b/compat/mingw.c index b3af992def50bf..06194706419b02 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2980,7 +2980,7 @@ int link(const char *oldpath, const char *newpath) return 0; } -int symlink(const char *target, const char *link) +int mingw_create_symlink(struct index_state *index UNUSED, const char *target, const char *link) { wchar_t wtarget[MAX_PATH], wlink[MAX_PATH]; int len; diff --git a/entry.c b/entry.c index 1c4f0f44070ea3..46cac258374d8e 100644 --- a/entry.c +++ b/entry.c @@ -323,7 +323,7 @@ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca state->istate->repo : the_repository) || to_tempfile) goto write_file_entry; - ret = symlink(new_blob, path); + ret = create_symlink(state->istate, new_blob, path); free(new_blob); if (ret) return error_errno("unable to create symlink %s", path); diff --git a/git-compat-util.h b/git-compat-util.h index a0f901ce793ed6..cd3f4c0baa15d6 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -350,6 +350,16 @@ static inline int git_has_dir_sep(const char *path) #define has_dir_sep(path) git_has_dir_sep(path) #endif +#ifndef create_symlink +struct index_state; +static inline int git_create_symlink(struct index_state *index UNUSED, + const char *target, const char *link) +{ + return symlink(target, link); +} +#define create_symlink git_create_symlink +#endif + #ifndef query_user_email #define query_user_email() NULL #endif diff --git a/refs/files-backend.c b/refs/files-backend.c index e8bed621598c50..e3797432559b2a 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -2249,7 +2249,7 @@ static int create_ref_symlink(struct ref_lock *lock, const char *target) ref_path = get_locked_file_path(&lock->lk); unlink(ref_path); - ret = symlink(target, ref_path); + ret = create_symlink(NULL, target, ref_path); free(ref_path); if (ret) diff --git a/setup.c b/setup.c index 61bcdafe82a20b..305ee7f662b55f 100644 --- a/setup.c +++ b/setup.c @@ -2383,7 +2383,7 @@ static void copy_templates_1(struct repository *repo, if (strbuf_readlink(&lnk, template_path->buf, st_template.st_size) < 0) die_errno(_("cannot readlink '%s'"), template_path->buf); - if (symlink(lnk.buf, path->buf)) + if (create_symlink(NULL, lnk.buf, path->buf)) die_errno(_("cannot symlink '%s' '%s'"), lnk.buf, path->buf); strbuf_release(&lnk); @@ -2669,7 +2669,7 @@ static int create_default_files(struct repository *repo, repo_git_path_replace(repo, &path, "tXXXXXX"); if (!close(xmkstemp(path.buf)) && !unlink(path.buf) && - !symlink("testing", path.buf) && + !create_symlink(NULL, "testing", path.buf) && !lstat(path.buf, &st1) && S_ISLNK(st1.st_mode)) unlink(path.buf); /* good */ From 589afeb2c2664f2103571b3be3977d42a71a7d03 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 18:07:31 +0200 Subject: [PATCH 097/229] combine-diff: stop truncating combined-diff blob sizes on Windows Continue the size_t evacuation. With buffer_is_binary() widened in the prior commit, every consumer that the size flows into in combine-diff.c is size_t-ready, so widen grab_blob()'s out-param outright and move the matching locals at its three call sites together. grab_blob()'s body collapses to a direct odb_read_object(&size) since the bridge variable is no longer needed. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- combine-diff.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/combine-diff.c b/combine-diff.c index 717d5376888f8e..09dca8fee9165a 100644 --- a/combine-diff.c +++ b/combine-diff.c @@ -304,7 +304,7 @@ static struct lline *coalesce_lines(struct lline *base, int *lenbase, static char *grab_blob(struct repository *r, const struct object_id *oid, unsigned int mode, - unsigned long *size, struct userdiff_driver *textconv, + size_t *size, struct userdiff_driver *textconv, const char *path) { char *blob; @@ -325,9 +325,7 @@ static char *grab_blob(struct repository *r, *size = fill_textconv(r, textconv, df, &blob); free_filespec(df); } else { - size_t size_st = 0; - blob = odb_read_object(r->objects, oid, &type, &size_st); - *size = cast_size_t_to_ulong(size_st); + blob = odb_read_object(r->objects, oid, &type, size); if (!blob) die(_("unable to read %s"), oid_to_hex(oid)); if (type != OBJ_BLOB) @@ -431,7 +429,7 @@ static void combine_diff(struct repository *r, xdemitconf_t xecfg; mmfile_t parent_file; struct combine_diff_state state; - unsigned long sz; + size_t sz; if (result_deleted) return; /* result deleted */ @@ -1015,7 +1013,7 @@ static void show_patch_diff(struct combine_diff_path *elem, int num_parent, struct rev_info *rev) { struct diff_options *opt = &rev->diffopt; - unsigned long result_size, cnt, lno; + size_t result_size, cnt, lno; int result_deleted = 0; char *result, *cp; struct sline *sline; /* survived lines */ @@ -1134,7 +1132,7 @@ static void show_patch_diff(struct combine_diff_path *elem, int num_parent, is_binary = buffer_is_binary(result, result_size); for (i = 0; !is_binary && i < num_parent; i++) { char *buf; - unsigned long size; + size_t size; buf = grab_blob(opt->repo, &elem->parent[i].oid, elem->parent[i].mode, From 11a43f1a6a81f741afc613177b9732517f643d51 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 23 Sep 2024 15:38:25 -0400 Subject: [PATCH 098/229] survey: add --top= option and config The 'git survey' builtin provides several detail tables, such as "top files by on-disk size". The size of these tables defaults to 10, currently. Allow the user to specify this number via a new --top= option or the new survey.top config key. Signed-off-by: Derrick Stolee Signed-off-by: Johannes Schindelin --- Documentation/config/survey.adoc | 3 +++ builtin/survey.c | 22 ++++++++++++++-------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/Documentation/config/survey.adoc b/Documentation/config/survey.adoc index c1b0f852a1250e..9e594a2092f225 100644 --- a/Documentation/config/survey.adoc +++ b/Documentation/config/survey.adoc @@ -8,4 +8,7 @@ survey.*:: This boolean value implies the `--[no-]verbose` option. progress:: This boolean value implies the `--[no-]progress` option. + top:: + This integer value implies `--top=`, specifying the + number of entries in the detail tables. -- diff --git a/builtin/survey.c b/builtin/survey.c index a0a3966295340f..936a27d0ca9784 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -40,6 +40,7 @@ static struct survey_refs_wanted default_ref_options = { struct survey_opts { int verbose; int show_progress; + int top_nr; struct survey_refs_wanted refs; }; @@ -548,6 +549,10 @@ static int survey_load_config_cb(const char *var, const char *value, ctx->opts.show_progress = git_config_bool(var, value); return 0; } + if (!strcmp(var, "survey.top")) { + ctx->opts.top_nr = git_config_bool(var, value); + return 0; + } return git_default_config(var, value, cctx, pvoid); } @@ -794,8 +799,6 @@ static int survey_objects_path_walk_fn(const char *path, static void initialize_report(struct survey_context *ctx) { - const int top_limit = 100; - CALLOC_ARRAY(ctx->report.by_type, REPORT_TYPE_COUNT); ctx->report.by_type[REPORT_TYPE_COMMIT].label = xstrdup(_("Commits")); ctx->report.by_type[REPORT_TYPE_TREE].label = xstrdup(_("Trees")); @@ -804,21 +807,21 @@ static void initialize_report(struct survey_context *ctx) CALLOC_ARRAY(ctx->report.top_paths_by_count, REPORT_TYPE_COUNT); init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_TREE], - top_limit, _("TOP DIRECTORIES BY COUNT"), cmp_by_nr); + ctx->opts.top_nr, _("TOP DIRECTORIES BY COUNT"), cmp_by_nr); init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_BLOB], - top_limit, _("TOP FILES BY COUNT"), cmp_by_nr); + ctx->opts.top_nr, _("TOP FILES BY COUNT"), cmp_by_nr); CALLOC_ARRAY(ctx->report.top_paths_by_disk, REPORT_TYPE_COUNT); init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_TREE], - top_limit, _("TOP DIRECTORIES BY DISK SIZE"), cmp_by_disk_size); + ctx->opts.top_nr, _("TOP DIRECTORIES BY DISK SIZE"), cmp_by_disk_size); init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_BLOB], - top_limit, _("TOP FILES BY DISK SIZE"), cmp_by_disk_size); + ctx->opts.top_nr, _("TOP FILES BY DISK SIZE"), cmp_by_disk_size); CALLOC_ARRAY(ctx->report.top_paths_by_inflate, REPORT_TYPE_COUNT); init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE], - top_limit, _("TOP DIRECTORIES BY INFLATED SIZE"), cmp_by_inflated_size); + ctx->opts.top_nr, _("TOP DIRECTORIES BY INFLATED SIZE"), cmp_by_inflated_size); init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB], - top_limit, _("TOP FILES BY INFLATED SIZE"), cmp_by_inflated_size); + ctx->opts.top_nr, _("TOP FILES BY INFLATED SIZE"), cmp_by_inflated_size); } static void survey_phase_objects(struct survey_context *ctx) @@ -869,6 +872,7 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor .opts = { .verbose = 0, .show_progress = -1, /* defaults to isatty(2) */ + .top_nr = 10, .refs.want_all_refs = -1, @@ -884,6 +888,8 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor static struct option survey_options[] = { OPT__VERBOSE(&ctx.opts.verbose, N_("verbose output")), OPT_BOOL(0, "progress", &ctx.opts.show_progress, N_("show progress")), + OPT_INTEGER('n', "top", &ctx.opts.top_nr, + N_("number of entries to include in detail tables")), OPT_BOOL_F(0, "all-refs", &ctx.opts.refs.want_all_refs, N_("include all refs"), PARSE_OPT_NONEG), From b0282e69eda258855544998431f2ea7d923ea0d1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 6 Jun 2026 01:31:22 +0200 Subject: [PATCH 099/229] git-survey: announce the upcoming pivot into `git repo structure` `git survey` started life as an experimental scale-measurement tool; the preceding commits give `git repo structure` the path-level detail tables and ref-scoping mechanism that were `git survey`'s main draw, so the two now overlap substantially. Plan the migration explicitly: add a short notice at the top of the description making clear which of `git survey`'s knobs map to which `git repo structure` option, and state that a future release will turn `git survey` into a thin shim over `git repo structure`. Putting the notice in the description (rather than only the synopsis) ensures it shows up in `git help survey` rendering before the reader sees any option specifics, so an operator skimming the page learns about the replacement before adopting any survey-specific flags. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- Documentation/git-survey.adoc | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Documentation/git-survey.adoc b/Documentation/git-survey.adoc index 44f3a0568b7697..349560eaa7ec28 100644 --- a/Documentation/git-survey.adoc +++ b/Documentation/git-survey.adoc @@ -3,16 +3,23 @@ git-survey(1) NAME ---- -git-survey - EXPERIMENTAL: Measure various repository dimensions of scale +git-survey - DEPRECATED: Measure various repository dimensions of scale SYNOPSIS -------- [verse] -(EXPERIMENTAL!) 'git survey' +(DEPRECATED!) 'git survey' DESCRIPTION ----------- +NOTE: `git survey` is being superseded by `git repo structure`. New +deployments and new features should use `git repo structure`; its +`--ref-filter=` option subsumes the various `--branches`, +`--tags`, and `--remotes` flags here, and `--top=` provides the +same detail tables. A future release will turn `git survey` into a +thin shim over `git repo structure`. See linkgit:git-repo[1]. + Survey the repository and measure various dimensions of scale. As repositories grow to "monorepo" size, certain data shapes can cause From 71028039b02a03e72e771d317b2ccfab6156a355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20A=C3=9Fhauer?= Date: Sun, 29 Dec 2024 11:48:34 +0100 Subject: [PATCH 100/229] t0301: actually test credential-cache on Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit 2406bf5 (Win32: detect unix socket support at runtime, 2024-04-03) introduced a runtime detection for whether the operating system supports unix sockets for Windows, but a mistake snuck into the tests. When building and testing Git without NO_UNIX_SOCKETS we currently skip t0301-credential-cache on Windows if unix sockets are supported and run the tests if they aren't. Flip that logic to actually work the way it was intended. Signed-off-by: Matthias Aßhauer Signed-off-by: Johannes Schindelin --- t/t0301-credential-cache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t0301-credential-cache.sh b/t/t0301-credential-cache.sh index 6f7cfd9e33f633..a14032626192d0 100755 --- a/t/t0301-credential-cache.sh +++ b/t/t0301-credential-cache.sh @@ -12,7 +12,7 @@ test -z "$NO_UNIX_SOCKETS" || { if test_have_prereq MINGW then service_running=$(sc query afunix | grep "4 RUNNING") - test -z "$service_running" || { + test -n "$service_running" || { skip_all='skipping credential-cache tests, unix sockets not available' test_done } From 1693890fa05abfd2c7f22d0ec2edbef2acd21555 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 26 Nov 2025 19:18:35 +0100 Subject: [PATCH 101/229] http: warn if might have failed because of NTLM The new default of Git is to disable NTLM authentication by default. To help users find the escape hatch of that config setting, should they need it, suggest it when the authentication failed and the server had offered NTLM, i.e. if re-enabling it would fix the problem. Helped-by: Patrick Steinhardt Signed-off-by: Johannes Schindelin --- http.c | 11 +++++++++++ t/t5563-simple-http-auth.sh | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/http.c b/http.c index 4844de1d3eed7a..5742a17a07ad19 100644 --- a/http.c +++ b/http.c @@ -1978,6 +1978,17 @@ static int handle_curl_result(struct slot_results *results) credential_reject(the_repository, &http_auth); if (always_auth_proactively()) http_proactive_auth = PROACTIVE_AUTH_NONE; + if ((results->auth_avail & CURLAUTH_NTLM) && + !(http_auth_any & CURLAUTH_NTLM)) { + warning(_("Due to its cryptographic weaknesses, " + "NTLM authentication has been\n" + "disabled in Git by default. You can " + "re-enable it for trusted servers\n" + "by running:\n\n" + "git config set " + "http.%s://%s.allowNTLMAuth true"), + http_auth.protocol, http_auth.host); + } return HTTP_NOAUTH; } else { if (curl_empty_auth == -1 && diff --git a/t/t5563-simple-http-auth.sh b/t/t5563-simple-http-auth.sh index 965c2fb71637c8..bd3b697cfd9050 100755 --- a/t/t5563-simple-http-auth.sh +++ b/t/t5563-simple-http-auth.sh @@ -776,7 +776,8 @@ test_expect_success NTLM 'access using NTLM auth' ' test_config_global credential.helper test-helper && test_must_fail env GIT_TRACE_CURL=1 git \ - ls-remote "$HTTPD_URL/ntlm_auth/repo.git" && + ls-remote "$HTTPD_URL/ntlm_auth/repo.git" 2>err && + test_grep "allowNTLMAuth" err && GIT_TRACE_CURL=1 git -c http.$HTTPD_URL.allowNTLMAuth=true \ ls-remote "$HTTPD_URL/ntlm_auth/repo.git" ' From ec85238fe00d5811687b04266dd4ec34b0faf1c9 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 9 May 2015 02:11:48 +0200 Subject: [PATCH 102/229] compat/terminal.c: only use the Windows console if bash 'read -r' fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Accessing the Windows console through the special CONIN$ / CONOUT$ devices doesn't work properly for non-ASCII usernames an passwords. It also doesn't work for terminal emulators that hide the native console window (such as mintty), and 'TERM=xterm*' is not necessarily a reliable indicator for such terminals. The new shell_prompt() function, on the other hand, works fine for both MSys1 and MSys2, in native console windows as well as mintty, and properly supports Unicode. It just needs bash on the path (for 'read -s', which is bash-specific). On Windows, try to use the shell to read from the terminal. If that fails with ENOENT (i.e. bash was not found), use CONIN/OUT as fallback. Note: To test this, create a UTF-8 credential file with non-ASCII chars, e.g. in git-bash: 'echo url=http://täst.com > cred.txt'. Then in git-cmd, 'git credential fill --- compat/terminal.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compat/terminal.c b/compat/terminal.c index cdcde283644e41..a89c5cd9ccf604 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -434,6 +434,7 @@ static char *shell_prompt(const char *prompt, int echo) strvec_pushv(&child.args, read_input); child.in = -1; child.out = -1; + child.silent_exec_failure = 1; if (start_command(&child)) return NULL; @@ -477,11 +478,14 @@ char *git_terminal_prompt(const char *prompt, int echo) static struct strbuf buf = STRBUF_INIT; int r; FILE *input_fh, *output_fh; + #ifdef GIT_WINDOWS_NATIVE - const char *term = getenv("TERM"); - if (term && starts_with(term, "xterm")) - return shell_prompt(prompt, echo); + /* try shell_prompt first, fall back to CONIN/OUT if bash is missing */ + char *result = shell_prompt(prompt, echo); + if (result || errno != ENOENT) + return result; + #endif input_fh = fopen(INPUT_PATH, "r" FORCE_TEXT); From 38c0f8b04249eebfcf26f8575513b7815a0f8608 Mon Sep 17 00:00:00 2001 From: JiSeop Moon Date: Mon, 23 Apr 2018 22:35:26 +0200 Subject: [PATCH 103/229] mingw: move the file_attr_to_st_mode() function definition In preparation for making this function a bit more complicated (to allow for special-casing the `ContainerMappedDirectories` in Windows containers, which look like a symbolic link, but are not), let's move it out of the header. Signed-off-by: JiSeop Moon Signed-off-by: Johannes Schindelin --- compat/mingw.c | 14 ++++++++++++++ compat/win32.h | 14 +------------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index e6e85e4be4028d..9b6ad28b05243d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3892,3 +3892,17 @@ int is_inside_windows_container(void) return inside_container; } + +int file_attr_to_st_mode (DWORD attr, DWORD tag) +{ + int fMode = S_IREAD; + if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && tag == IO_REPARSE_TAG_SYMLINK) + fMode |= S_IFLNK; + else if (attr & FILE_ATTRIBUTE_DIRECTORY) + fMode |= S_IFDIR; + else + fMode |= S_IFREG; + if (!(attr & FILE_ATTRIBUTE_READONLY)) + fMode |= S_IWRITE; + return fMode; +} diff --git a/compat/win32.h b/compat/win32.h index 671bcc81f93351..52169ae19f4371 100644 --- a/compat/win32.h +++ b/compat/win32.h @@ -6,19 +6,7 @@ #include #endif -static inline int file_attr_to_st_mode (DWORD attr, DWORD tag) -{ - int fMode = S_IREAD; - if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && tag == IO_REPARSE_TAG_SYMLINK) - fMode |= S_IFLNK; - else if (attr & FILE_ATTRIBUTE_DIRECTORY) - fMode |= S_IFDIR; - else - fMode |= S_IFREG; - if (!(attr & FILE_ATTRIBUTE_READONLY)) - fMode |= S_IWRITE; - return fMode; -} +extern int file_attr_to_st_mode (DWORD attr, DWORD tag); static inline int get_file_attr(const char *fname, WIN32_FILE_ATTRIBUTE_DATA *fdata) { From 0c6a848e007a03403ddc1e640028c1b38598d3cb Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 26 Oct 2018 11:51:51 +0200 Subject: [PATCH 104/229] mingw: allow to specify the symlink type in .gitattributes On Windows, symbolic links have a type: a "file symlink" must point at a file, and a "directory symlink" must point at a directory. If the type of symlink does not match its target, it doesn't work. Git does not record the type of symlink in the index or in a tree. On checkout it'll guess the type, which only works if the target exists at the time the symlink is created. This may often not be the case, for example when the link points at a directory inside a submodule. By specifying `symlink=file` or `symlink=dir` the user can specify what type of symlink Git should create, so Git doesn't have to rely on unreliable heuristics. Signed-off-by: Bert Belder Signed-off-by: Johannes Schindelin --- Documentation/gitattributes.adoc | 30 ++++++++++++++++ compat/mingw.c | 60 ++++++++++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/Documentation/gitattributes.adoc b/Documentation/gitattributes.adoc index 9a50d6338b8e23..ecbf61253d8a17 100644 --- a/Documentation/gitattributes.adoc +++ b/Documentation/gitattributes.adoc @@ -403,6 +403,36 @@ sign `$` upon checkout. Any byte sequence that begins with with `$Id$` upon check-in. +`symlink` +^^^^^^^^^ + +On Windows, symbolic links have a type: a "file symlink" must point at +a file, and a "directory symlink" must point at a directory. If the +type of symlink does not match its target, it doesn't work. + +Git does not record the type of symlink in the index or in a tree. On +checkout it'll guess the type, which only works if the target exists +at the time the symlink is created. This may often not be the case, +for example when the link points at a directory inside a submodule. + +The `symlink` attribute allows you to explicitly set the type of symlink +to `file` or `dir`, so Git doesn't have to guess. If you have a set of +symlinks that point at other files, you can do: + +------------------------ +*.gif symlink=file +------------------------ + +To tell Git that a symlink points at a directory, use: + +------------------------ +tools_folder symlink=dir +------------------------ + +The `symlink` attribute is ignored on platforms other than Windows, +since they don't distinguish between different types of symlinks. + + `filter` ^^^^^^^^ diff --git a/compat/mingw.c b/compat/mingw.c index 06194706419b02..02b488bd6abafc 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -4,6 +4,7 @@ #include "git-compat-util.h" #include "abspath.h" #include "alloc.h" +#include "attr.h" #include "config.h" #include "dir.h" #include "environment.h" @@ -2980,7 +2981,38 @@ int link(const char *oldpath, const char *newpath) return 0; } -int mingw_create_symlink(struct index_state *index UNUSED, const char *target, const char *link) +enum symlink_type { + SYMLINK_TYPE_UNSPECIFIED = 0, + SYMLINK_TYPE_FILE, + SYMLINK_TYPE_DIRECTORY, +}; + +static enum symlink_type check_symlink_attr(struct index_state *index, const char *link) +{ + static struct attr_check *check; + const char *value; + + if (!index) + return SYMLINK_TYPE_UNSPECIFIED; + + if (!check) + check = attr_check_initl("symlink", NULL); + + git_check_attr(index, link, check); + + value = check->items[0].value; + if (ATTR_UNSET(value)) + return SYMLINK_TYPE_UNSPECIFIED; + if (!strcmp(value, "file")) + return SYMLINK_TYPE_FILE; + if (!strcmp(value, "dir") || !strcmp(value, "directory")) + return SYMLINK_TYPE_DIRECTORY; + + warning(_("ignoring invalid symlink type '%s' for '%s'"), value, link); + return SYMLINK_TYPE_UNSPECIFIED; +} + +int mingw_create_symlink(struct index_state *index, const char *target, const char *link) { wchar_t wtarget[MAX_PATH], wlink[MAX_PATH]; int len; @@ -3000,7 +3032,31 @@ int mingw_create_symlink(struct index_state *index UNUSED, const char *target, c if (wtarget[len] == '/') wtarget[len] = '\\'; - return create_phantom_symlink(wtarget, wlink); + switch (check_symlink_attr(index, link)) { + case SYMLINK_TYPE_UNSPECIFIED: + /* Create a phantom symlink: it is initially created as a file + * symlink, but may change to a directory symlink later if/when + * the target exists. */ + return create_phantom_symlink(wtarget, wlink); + case SYMLINK_TYPE_FILE: + if (!CreateSymbolicLinkW(wlink, wtarget, symlink_file_flags)) + break; + return 0; + case SYMLINK_TYPE_DIRECTORY: + if (!CreateSymbolicLinkW(wlink, wtarget, + symlink_directory_flags)) + break; + /* There may be dangling phantom symlinks that point at this + * one, which should now morph into directory symlinks. */ + process_phantom_symlinks(); + return 0; + default: + BUG("unhandled symlink type"); + } + + /* CreateSymbolicLinkW failed. */ + errno = err_win_to_posix(GetLastError()); + return -1; } int readlink(const char *path, char *buf, size_t bufsiz) From 4289a5e6036d9d68c3ae90b01a483631d1012fd5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 7 Dec 2018 13:39:30 +0100 Subject: [PATCH 105/229] clean: do not traverse mount points It seems to be not exactly rare on Windows to install NTFS junction points (the equivalent of "bind mounts" on Linux/Unix) in worktrees, e.g. to map some development tools into a subdirectory. In such a scenario, it is pretty horrible if `git clean -dfx` traverses into the mapped directory and starts to "clean up". Let's just not do that. Let's make sure before we traverse into a directory that it is not a mount point (or junction). This addresses https://github.com/git-for-windows/git/issues/607 Signed-off-by: Johannes Schindelin --- builtin/clean.c | 14 ++++++++++++++ compat/mingw.c | 22 ++++++++++++++++++++++ compat/mingw.h | 3 +++ git-compat-util.h | 4 ++++ path.c | 39 +++++++++++++++++++++++++++++++++++++++ path.h | 1 + t/t7300-clean.sh | 9 +++++++++ 7 files changed, 92 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index 1d5e7e5366bf09..e4f2d56d3210ba 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -41,6 +41,8 @@ static const char *msg_remove = N_("Removing %s\n"); static const char *msg_would_remove = N_("Would remove %s\n"); static const char *msg_skip_git_dir = N_("Skipping repository %s\n"); static const char *msg_would_skip_git_dir = N_("Would skip repository %s\n"); +static const char *msg_skip_mount_point = N_("Skipping mount point %s\n"); +static const char *msg_would_skip_mount_point = N_("Would skip mount point %s\n"); static const char *msg_warn_remove_failed = N_("failed to remove %s"); static const char *msg_warn_lstat_failed = N_("could not lstat %s\n"); static const char *msg_skip_cwd = N_("Refusing to remove current working directory\n"); @@ -185,6 +187,18 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, goto out; } + if (is_mount_point(path)) { + if (!quiet) { + quote_path(path->buf, prefix, "ed, 0); + printf(dry_run ? + _(msg_would_skip_mount_point) : + _(msg_skip_mount_point), quoted.buf); + } + *dir_gone = 0; + + goto out; + } + dir = opendir(path->buf); if (!dir) { /* an empty dir could be removed even if it is unreadble */ diff --git a/compat/mingw.c b/compat/mingw.c index 02b488bd6abafc..bd23250bb6d2d4 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3135,6 +3135,28 @@ pid_t waitpid(pid_t pid, int *status, int options) return -1; } +int mingw_is_mount_point(struct strbuf *path) +{ + WIN32_FIND_DATAW findbuf = { 0 }; + HANDLE handle; + wchar_t wfilename[MAX_PATH]; + int wlen = xutftowcs_path(wfilename, path->buf); + if (wlen < 0) + die(_("could not get long path for '%s'"), path->buf); + + /* remove trailing slash, if any */ + if (wlen > 0 && wfilename[wlen - 1] == L'/') + wfilename[--wlen] = L'\0'; + + handle = FindFirstFileW(wfilename, &findbuf); + if (handle == INVALID_HANDLE_VALUE) + return 0; + FindClose(handle); + + return (findbuf.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) && + (findbuf.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT); +} + int xutftowcsn(wchar_t *wcs, const char *utfs, size_t wcslen, int utflen) { int upos = 0, wpos = 0; diff --git a/compat/mingw.h b/compat/mingw.h index df02aeb632d8c3..92237e14c4a87c 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -36,6 +36,9 @@ static inline void convert_slashes(char *path) if (*path == '\\') *path = '/'; } +struct strbuf; +int mingw_is_mount_point(struct strbuf *path); +#define is_mount_point mingw_is_mount_point #define PATH_SEP ';' char *mingw_query_user_email(void); #define query_user_email mingw_query_user_email diff --git a/git-compat-util.h b/git-compat-util.h index cd3f4c0baa15d6..51f1b2ecce338b 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -350,6 +350,10 @@ static inline int git_has_dir_sep(const char *path) #define has_dir_sep(path) git_has_dir_sep(path) #endif +#ifndef is_mount_point +#define is_mount_point is_mount_point_via_stat +#endif + #ifndef create_symlink struct index_state; static inline int git_create_symlink(struct index_state *index UNUSED, diff --git a/path.c b/path.c index 7078417d92af14..19293484f106c2 100644 --- a/path.c +++ b/path.c @@ -1327,6 +1327,45 @@ char *strip_path_suffix(const char *path, const char *suffix) return offset == -1 ? NULL : xstrndup(path, offset); } +int is_mount_point_via_stat(struct strbuf *path) +{ + size_t len = path->len; + dev_t current_dev; + struct stat st; + + if (!strcmp("/", path->buf)) + return 1; + + strbuf_addstr(path, "/."); + if (lstat(path->buf, &st)) { + /* + * If we cannot access the current directory, we cannot say + * that it is a bind mount. + */ + strbuf_setlen(path, len); + return 0; + } + current_dev = st.st_dev; + + /* Now look at the parent directory */ + strbuf_addch(path, '.'); + if (lstat(path->buf, &st)) { + /* + * If we cannot access the parent directory, we cannot say + * that it is a bind mount. + */ + strbuf_setlen(path, len); + return 0; + } + strbuf_setlen(path, len); + + /* + * If the device ID differs between current and parent directory, + * then it is a bind mount. + */ + return current_dev != st.st_dev; +} + int daemon_avoid_alias(const char *p) { int sl, ndot; diff --git a/path.h b/path.h index ee4a91c9c512f8..8f365de55b275a 100644 --- a/path.h +++ b/path.h @@ -162,6 +162,7 @@ int normalize_path_copy(char *dst, const char *src); int strbuf_normalize_path(struct strbuf *src); int longest_ancestor_length(const char *path, struct string_list *prefixes); char *strip_path_suffix(const char *path, const char *suffix); +int is_mount_point_via_stat(struct strbuf *path); int daemon_avoid_alias(const char *path); /* diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index 13ad3eec772e37..af9c05255ca040 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -800,4 +800,13 @@ test_expect_success 'traverse into directories that may have ignored entries' ' ) ' +test_expect_success MINGW 'clean does not traverse mount points' ' + mkdir target && + >target/dont-clean-me && + git init with-mountpoint && + cmd //c "mklink /j with-mountpoint\\mountpoint target" && + git -C with-mountpoint clean -dfx && + test_path_is_file target/dont-clean-me +' + test_done From 1f210b571a28ef3aaae048ae91c29746c8865e44 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 19:26:00 +0200 Subject: [PATCH 106/229] diff: widen textconv_object() size out-param to size_t Continue the size_t evacuation. textconv_object() fills its out-parameter from fill_textconv()'s size_t return through an unsigned long*; widen the API to match, then take advantage of the new shape where callers can. cat-file's 'c' and batch-mode 'c' branches lose their size_ul bridge variables (one site becomes a direct call, the other collapses an if/else into a single negated condition that reads as "try textconv, fall back to a raw read"). blame.c likewise drops the file_size_st bridge in fill_origin_blob() and hoists final_buf_size_st to bracket both branches in setup_scoreboard(). The latter keeps a cast_size_t_to_ulong() shim because struct blame_scoreboard.final_buf_size is still unsigned long; that field is its own topic. log.c just widens its local from unsigned long to size_t. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- blame.c | 21 ++++++++------------- builtin/cat-file.c | 13 ++++--------- builtin/log.c | 2 +- diff.c | 2 +- diff.h | 2 +- 5 files changed, 15 insertions(+), 25 deletions(-) diff --git a/blame.c b/blame.c index 3189bcdabaa886..9686757ed3c08a 100644 --- a/blame.c +++ b/blame.c @@ -240,7 +240,7 @@ static struct commit *fake_working_tree_commit(struct repository *r, struct stat st; const char *read_from; char *buf_ptr; - unsigned long buf_len; + size_t buf_len; if (contents_from) { if (stat(contents_from, &st) < 0) @@ -1036,20 +1036,17 @@ static void fill_origin_blob(struct diff_options *opt, { if (!o->file.ptr) { enum object_type type; - unsigned long file_size; + size_t file_size; (*num_read_blob)++; if (opt->flags.allow_textconv && textconv_object(opt->repo, o->path, o->mode, &o->blob_oid, 1, &file->ptr, &file_size)) ; - else { - size_t file_size_st = 0; + else file->ptr = odb_read_object(the_repository->objects, &o->blob_oid, &type, - &file_size_st); - file_size = cast_size_t_to_ulong(file_size_st); - } + &file_size); file->size = file_size; if (!file->ptr) @@ -2931,22 +2928,20 @@ void setup_scoreboard(struct blame_scoreboard *sb, sb->final_buf_size = o->file.size; } else { + size_t final_buf_size_st = 0; o = get_origin(sb->final, sb->path); if (fill_blob_sha1_and_mode(sb->repo, o)) die(_("no such path %s in %s"), sb->path, final_commit_name); if (sb->revs->diffopt.flags.allow_textconv && textconv_object(sb->repo, sb->path, o->mode, &o->blob_oid, 1, (char **) &sb->final_buf, - &sb->final_buf_size)) + &final_buf_size_st)) ; - else { - size_t final_buf_size_st = 0; + else sb->final_buf = odb_read_object(the_repository->objects, &o->blob_oid, &type, &final_buf_size_st); - sb->final_buf_size = - cast_size_t_to_ulong(final_buf_size_st); - } + sb->final_buf_size = cast_size_t_to_ulong(final_buf_size_st); if (!sb->final_buf) die(_("cannot read blob %s for path %s"), diff --git a/builtin/cat-file.c b/builtin/cat-file.c index 8870a210ec2e94..e8fdf1a51f5caf 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -202,11 +202,9 @@ static int cat_one_file(int opt, const char *exp_type, const char *obj_name) case 'c': { - unsigned long size_ul = 0; int textconv_ret = textconv_object(the_repository, path, obj_context.mode, &oid, 1, - &buf, &size_ul); - size = size_ul; + &buf, &size); if (textconv_ret) break; } @@ -457,12 +455,9 @@ static void print_object_or_die(struct batch_options *opt, struct expand_data *d oid_to_hex(oid), data->rest); } else if (opt->transform_mode == 'c') { enum object_type type; - unsigned long size_ul = 0; - if (textconv_object(the_repository, - data->rest, 0100644, oid, - 1, &contents, &size_ul)) - size = size_ul; - else + if (!textconv_object(the_repository, + data->rest, 0100644, oid, + 1, &contents, &size)) contents = odb_read_object(the_repository->objects, oid, &type, &size); if (!contents) diff --git a/builtin/log.c b/builtin/log.c index f3a4ed0c5c5769..cae2a8633a4373 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -585,7 +585,7 @@ static int show_blob_object(const struct object_id *oid, struct rev_info *rev, c struct object_id oidc; struct object_context obj_context = {0}; char *buf; - unsigned long size; + size_t size; fflush(rev->diffopt.file); if (!rev->diffopt.flags.textconv_set_via_cmdline || diff --git a/diff.c b/diff.c index 1cbd6c255e4538..5e7834851d0852 100644 --- a/diff.c +++ b/diff.c @@ -8049,7 +8049,7 @@ int textconv_object(struct repository *r, const struct object_id *oid, int oid_valid, char **buf, - unsigned long *buf_size) + size_t *buf_size) { struct diff_filespec *df; struct userdiff_driver *textconv; diff --git a/diff.h b/diff.h index e0b58a8105fe8f..095ece341c895e 100644 --- a/diff.h +++ b/diff.h @@ -804,7 +804,7 @@ int textconv_object(struct repository *repo, const char *path, unsigned mode, const struct object_id *oid, int oid_valid, - char **buf, unsigned long *buf_size); + char **buf, size_t *buf_size); int parse_rename_score(const char **cp_p); From 7c10cbc03307e4c99981b5c6e90a558a58db2ba9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 15:00:18 +0200 Subject: [PATCH 107/229] pack-bitmap: stop truncating blob sizes used by --filter=blob:limit Same theme as the preceding pack-objects series: get_size_by_pos() returns an unsigned long but reads its size out of packed_object_info() / odb_read_object_info_extended() via a size_t out-parameter, so on Windows it would silently truncate the very sizes filter_bitmap_blob_limit() then compares against the --filter=blob:limit threshold to decide which blobs to elide from the bitmap-backed traversal. Drop the cast_size_t_to_ulong() and return size_t directly. The two callers' limit comparison promotes to size_t cleanly. limit itself stays unsigned long; it is part of a filter API ripple of its own. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- pack-bitmap.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pack-bitmap.c b/pack-bitmap.c index d3c7997fb3c1cf..e77cad81f85073 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -1867,8 +1867,8 @@ static void filter_bitmap_blob_none(struct bitmap_index *bitmap_git, OBJ_BLOB); } -static unsigned long get_size_by_pos(struct bitmap_index *bitmap_git, - uint32_t pos) +static size_t get_size_by_pos(struct bitmap_index *bitmap_git, + uint32_t pos) { size_t size; struct object_info oi = OBJECT_INFO_INIT; @@ -1905,7 +1905,7 @@ static unsigned long get_size_by_pos(struct bitmap_index *bitmap_git, die(_("unable to get size of %s"), oid_to_hex(&obj->oid)); } - return cast_size_t_to_ulong(size); + return size; } static void filter_bitmap_blob_limit(struct bitmap_index *bitmap_git, From 39a3bb2ed52798ccc6785237cad8b71aad0cdd24 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 22 Nov 2023 22:57:38 +0100 Subject: [PATCH 108/229] win32: use native ANSI sequence processing, if possible Windows 10 version 1511 (also known as Anniversary Update), according to https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences introduced native support for ANSI sequence processing. This allows using colors from the entire 24-bit color range. All we need to do is test whether the console's "virtual processing support" can be enabled. If it can, we do not even need to start the `console_thread` to handle ANSI sequences. Or, almost all we need to do: When `console_thread()` does its work, it uses the Unicode-aware `write_console()` function to write to the Win32 Console, which supports Git for Windows' implicit convention that all text that is written is encoded in UTF-8. The same is not necessarily true if native ANSI sequence processing is used, as the output is then subject to the current code page. Let's ensure that the code page is set to `CP_UTF8` as long as Git writes to it. Signed-off-by: Johannes Schindelin --- compat/winansi.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/compat/winansi.c b/compat/winansi.c index 088734a1dfa8f7..d94448ceb01ae1 100644 --- a/compat/winansi.c +++ b/compat/winansi.c @@ -555,6 +555,49 @@ static void detect_msys_tty(int fd) #endif +static HANDLE std_console_handle; +static DWORD std_console_mode = ENABLE_VIRTUAL_TERMINAL_PROCESSING; +static UINT std_console_code_page = CP_UTF8; + +static void reset_std_console(void) +{ + if (std_console_mode != ENABLE_VIRTUAL_TERMINAL_PROCESSING) + SetConsoleMode(std_console_handle, std_console_mode); + if (std_console_code_page != CP_UTF8) + SetConsoleOutputCP(std_console_code_page); +} + +static int enable_virtual_processing(void) +{ + std_console_handle = GetStdHandle(STD_OUTPUT_HANDLE); + if (std_console_handle == INVALID_HANDLE_VALUE || + !GetConsoleMode(std_console_handle, &std_console_mode)) { + std_console_handle = GetStdHandle(STD_ERROR_HANDLE); + if (std_console_handle == INVALID_HANDLE_VALUE || + !GetConsoleMode(std_console_handle, &std_console_mode)) + return 0; + } + + std_console_code_page = GetConsoleOutputCP(); + if (std_console_code_page != CP_UTF8) + SetConsoleOutputCP(CP_UTF8); + if (!std_console_code_page) + std_console_code_page = CP_UTF8; + + atexit(reset_std_console); + + if (std_console_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING) + return 1; + + if (!SetConsoleMode(std_console_handle, + std_console_mode | + ENABLE_PROCESSED_OUTPUT | + ENABLE_VIRTUAL_TERMINAL_PROCESSING)) + return 0; + + return 1; +} + /* * Wrapper for isatty(). Most calls in the main git code * call isatty(1 or 2) to see if the instance is interactive @@ -593,6 +636,9 @@ void winansi_init(void) return; } + if (enable_virtual_processing()) + return; + /* create a named pipe to communicate with the console thread */ if (swprintf(name, ARRAY_SIZE(name) - 1, L"\\\\.\\pipe\\winansi%lu", GetCurrentProcessId()) < 0) From 213891894cd6fc2e5847c013114828429cb2a152 Mon Sep 17 00:00:00 2001 From: MinarKotonoha Date: Mon, 8 Apr 2024 16:41:10 +0800 Subject: [PATCH 109/229] common-main.c: fflush stdout buffer upon exit By default, the buffer type of Windows' `stdout` is unbuffered (_IONBF), and there is no need to manually fflush `stdout`. But some programs, such as the Windows Filtering Platform driver provided by the security software, may change the buffer type of `stdout` to full buffering. This nees `fflush(stdout)` to be called manually, otherwise there will be no output to `stdout`. Signed-off-by: MinarKotonoha Signed-off-by: Johannes Schindelin --- common-exit.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/common-exit.c b/common-exit.c index 1aaa538be3ed67..609f32abed8b53 100644 --- a/common-exit.c +++ b/common-exit.c @@ -11,6 +11,13 @@ static void check_bug_if_BUG(void) /* We wrap exit() to call common_exit() in git-compat-util.h */ int common_exit(const char *file, int line, int code) { + /* + * Windows Filtering Platform driver provided by the security software + * may change buffer type of stdout from _IONBF to _IOFBF. + * It will no output without fflush manually. + */ + fflush(stdout); + /* * For non-POSIX systems: Take the lowest 8 bits of the "code" * to e.g. turn -1 into 255. On a POSIX system this is From 7057ce4630d8a2acd10a532216391304e8f2e203 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 9 Apr 2024 16:50:56 +0200 Subject: [PATCH 110/229] t5601/t7406(mingw): do run tests with symlink support A long time ago, we decided to run tests in Git for Windows' SDK with the default `winsymlinks` mode: copying instead of linking. This is still the default mode of MSYS2 to this day. However, this is not how most users run Git for Windows: As the majority of Git for Windows' users seem to be on Windows 10 and newer, likely having enabled Developer Mode (which allows creating symbolic links without administrator privileges), they will run with symlink support enabled. This is the reason why it is crucial to get the fixes for CVE-2024-? to the users, and also why it is crucial to ensure that the test suite exercises the related test cases. This commit ensures the latter. Signed-off-by: Johannes Schindelin --- t/t5601-clone.sh | 10 ++++++++++ t/t7406-submodule-update.sh | 9 +++++++++ 2 files changed, 19 insertions(+) diff --git a/t/t5601-clone.sh b/t/t5601-clone.sh index 06f6121f82e526..5133442f8fc944 100755 --- a/t/t5601-clone.sh +++ b/t/t5601-clone.sh @@ -7,6 +7,16 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh +# This test script contains test cases that need to create symbolic links. To +# make sure that these test cases are exercised in Git for Windows, where (for +# historical reasons) `ln -s` creates copies by default, let's specifically ask +# for `ln -s` to create symbolic links whenever possible. +if test_have_prereq MINGW +then + MSYS=${MSYS+$MSYS }winsymlinks:nativestrict + export MSYS +fi + X= test_have_prereq !MINGW || X=.exe diff --git a/t/t7406-submodule-update.sh b/t/t7406-submodule-update.sh index 10adeabf0fcc85..f3b0353c78fcea 100755 --- a/t/t7406-submodule-update.sh +++ b/t/t7406-submodule-update.sh @@ -14,6 +14,15 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh +# This test script contains test cases that need to create symbolic links. To +# make sure that these test cases are exercised in Git for Windows, where (for +# historical reasons) `ln -s` creates copies by default, let's specifically ask +# for `ln -s` to create symbolic links whenever possible. +if test_have_prereq MINGW +then + MSYS=${MSYS+$MSYS }winsymlinks:nativestrict + export MSYS +fi compare_head() { From 529d97d4fb4c0f57f1fb1ab1499fd30b2325031d Mon Sep 17 00:00:00 2001 From: Ariel Lourenco Date: Tue, 2 Jul 2024 18:09:43 -0300 Subject: [PATCH 111/229] Fallback to AppData if XDG_CONFIG_HOME is unset In order to be a better Windows citizenship, Git should save its configuration files on AppData folder. This can enables git configuration files be replicated between machines using the same Microsoft account logon which would reduce the friction of setting up Git on new systems. Therefore, if %APPDATA%\Git\config exists, we use it; otherwise $HOME/.config/git/config is used. Signed-off-by: Ariel Lourenco --- path.c | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/path.c b/path.c index 7078417d92af14..988e102f14e4a4 100644 --- a/path.c +++ b/path.c @@ -1551,13 +1551,37 @@ char *xdg_config_home_for(const char *subdir, const char *filename) config_home = getenv("XDG_CONFIG_HOME"); if (config_home && *config_home) ret = mkpathdup("%s/%s/%s", config_home, subdir, filename); - else if ((home = getenv("HOME"))) - ret = mkpathdup("%s/.config/%s/%s", home, subdir, filename); - else - return NULL; + else { + home = getenv("HOME"); + ret = home && *home ? + mkpathdup("%s/.config/%s/%s", home, subdir, filename) : + NULL; + +#ifdef WIN32 + { + const char *appdata = getenv("APPDATA"); + + if (appdata && *appdata) { + char *appdata_config = + mkpathdup("%s/Git/%s", appdata, filename); + + if (file_exists(appdata_config)) { + if (ret && file_exists(ret)) + warning("'%s' was ignored because '%s' exists.", + ret, appdata_config); + free(ret); + ret = appdata_config; + } else { + free(appdata_config); + } + } + } +#endif + } #ifdef GIT_WINDOWS_NATIVE - convert_slashes(ret); + if (ret) + convert_slashes(ret); #endif return ret; } From 2fb16b3c5183f2b6261e06eabf7900d28dda993d Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 6 Sep 2024 14:16:13 -0400 Subject: [PATCH 112/229] revision: create mark_trees_uninteresting_dense() The sparse tree walk algorithm was created in d5d2e93577e (revision: implement sparse algorithm, 2019-01-16) and involves using the mark_trees_uninteresting_sparse() method. This method takes a repository and an oidset of tree IDs, some of which have the UNINTERESTING flag and some of which do not. Create a method that has an equivalent set of preconditions but uses a "dense" walk (recursively visits all reachable trees, as long as they have not previously been marked UNINTERESTING). This is an important difference from mark_tree_uninteresting(), which short-circuits if the given tree has the UNINTERESTING flag. A use of this method will be added in a later change, with a condition set whether the sparse or dense approach should be used. Signed-off-by: Derrick Stolee From 97b18379f4e7d83bf3a2e17e0b4eeccff374c3f2 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 1 Jul 2024 23:28:45 +0200 Subject: [PATCH 113/229] survey: clearly note the experimental nature in the output While this command is definitely something we _want_, chances are that upstreaming this will require substantial changes. We still want to be able to experiment with this before that, to focus on what we need out of this command: To assist with diagnosing issues with large repositories, as well as to help monitoring the growth and the associated painpoints of such repositories. To that end, we are about to integrate this command into `microsoft/git`, to get the tool into the hands of users who need it most, with the idea to iterate in close collaboration between these users and the developers familar with Git's internals. However, we will definitely want to avoid letting anybody have the impression that this command, its exact inner workings, as well as its output format, are anywhere close to stable. To make that fact utterly clear (and thereby protect the freedom to iterate and innovate freely before upstreaming the command), let's mark its output as experimental in all-caps, as the first thing we do. Signed-off-by: Johannes Schindelin --- builtin/survey.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/builtin/survey.c b/builtin/survey.c index 936a27d0ca9784..406e76eb098d03 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -17,6 +17,7 @@ #include "strvec.h" #include "tag.h" #include "trace2.h" +#include "color.h" static const char * const survey_usage[] = { N_("(EXPERIMENTAL!) git survey "), @@ -905,6 +906,11 @@ int cmd_survey(int argc, const char **argv, const char *prefix, struct repositor show_usage_with_options_if_asked(argc, argv, survey_usage, survey_options); + if (isatty(2)) + color_fprintf_ln(stderr, + want_color_fd(2, GIT_COLOR_AUTO) ? GIT_COLOR_YELLOW : "", + "(THIS IS EXPERIMENTAL, EXPECT THE OUTPUT FORMAT TO CHANGE!)"); + ctx.repo = repo; prepare_repo_settings(ctx.repo); From 9dce04eded9c8156aab695bd6beaa24866aec4de Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 6 Jun 2026 01:46:57 +0200 Subject: [PATCH 114/229] survey: turn into a thin shim over `git repo structure` `git survey` was an experimental scale-measurement tool whose distinctive features (ref-kind filters, top-N path tables) are now all available in `git repo structure`. With the path-level reporting in place (commits "repo: filter the structure scope via --ref-filter=" and "repo: report top-N paths by count, disk, and inflated size in structure"), there is no functionality `git survey` provides that `git repo structure` cannot. Replace the 764-line `git survey` implementation with a roughly hundred-line shim that: * Accepts the existing `git survey` command line so callers in scripts continue to parse without changes. * Emits a deprecation warning naming the replacement command, so interactive users learn about the migration target. * Translates the survey-specific knobs into the equivalent `git repo structure` invocation and re-execs the canonical command via `execv_git_cmd()`. Per-kind ref selectors fan out into the corresponding `refs/heads/*`, `refs/tags/*`, etc. `--ref-filter` patterns; `--top=` is forwarded directly; `--all-refs` becomes the absence of any `--ref-filter`. Two survey options have no `git repo structure` counterpart: `--verbose` controlled per-step trace output the new command does not emit, and `--detached` selected the detached HEAD which `git repo structure` does not enumerate separately. Both are silently accepted and produce a single warning each, so old invocations keep working while the absence of these knobs in `git repo structure` is made visible. Rewrite t8100 to assert the shim's contract: the deprecation warning is printed, the output is byte-identical to a corresponding `git repo structure` invocation, and the per-kind selector translation produces the right `--ref-filter` pattern. The preceding survey-specific output assertions (the multi-column plaintext tables) no longer apply, since `git repo structure`'s output format is now the canonical one and is covered by t1901. The `survey.*` configuration keys (`survey.top`, `survey.progress`, `survey.verbose`) are no longer honored by the shim. They were mirrored by the preceding `repo.structure.top` work for the most useful knob; users with `survey.top` set in config should migrate to `repo.structure.top`. This is a backward-incompatible removal documented by the deprecation notice in `git-survey.adoc`. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- Documentation/git-survey.adoc | 4 +- builtin/survey.c | 1007 +++------------------------------ t/t8100-git-survey.sh | 98 +--- 3 files changed, 116 insertions(+), 993 deletions(-) diff --git a/Documentation/git-survey.adoc b/Documentation/git-survey.adoc index 349560eaa7ec28..e97656b49dee75 100644 --- a/Documentation/git-survey.adoc +++ b/Documentation/git-survey.adoc @@ -17,8 +17,8 @@ NOTE: `git survey` is being superseded by `git repo structure`. New deployments and new features should use `git repo structure`; its `--ref-filter=` option subsumes the various `--branches`, `--tags`, and `--remotes` flags here, and `--top=` provides the -same detail tables. A future release will turn `git survey` into a -thin shim over `git repo structure`. See linkgit:git-repo[1]. +same detail tables. During the deprecation phase, `git survey` is +a thin shim over `git repo structure`. See linkgit:git-repo[1]. Survey the repository and measure various dimensions of scale. diff --git a/builtin/survey.c b/builtin/survey.c index 406e76eb098d03..d3b36b891404cd 100644 --- a/builtin/survey.c +++ b/builtin/survey.c @@ -1,934 +1,113 @@ #define USE_THE_REPOSITORY_VARIABLE #include "builtin.h" -#include "config.h" -#include "environment.h" -#include "hex.h" -#include "object.h" -#include "odb.h" -#include "object-name.h" +#include "color.h" +#include "exec-cmd.h" +#include "gettext.h" #include "parse-options.h" -#include "path-walk.h" -#include "progress.h" -#include "ref-filter.h" -#include "refs.h" -#include "revision.h" -#include "strbuf.h" #include "strvec.h" -#include "tag.h" -#include "trace2.h" -#include "color.h" static const char * const survey_usage[] = { - N_("(EXPERIMENTAL!) git survey "), + N_("(DEPRECATED!) git survey "), NULL, }; -struct survey_refs_wanted { - int want_all_refs; /* special override */ - - int want_branches; - int want_tags; - int want_remotes; - int want_detached; - int want_other; /* see FILTER_REFS_OTHERS -- refs/notes/, refs/stash/ */ -}; - -static struct survey_refs_wanted default_ref_options = { - .want_all_refs = 1, -}; - -struct survey_opts { - int verbose; - int show_progress; - int top_nr; - struct survey_refs_wanted refs; -}; - -struct survey_report_ref_summary { - size_t refs_nr; - size_t branches_nr; - size_t remote_refs_nr; - size_t tags_nr; - size_t tags_annotated_nr; - size_t others_nr; - size_t unknown_nr; -}; - -struct survey_report_object_summary { - size_t commits_nr; - size_t tags_nr; - size_t trees_nr; - size_t blobs_nr; -}; - -/** - * For some category given by 'label', count the number of objects - * that match that label along with the on-disk size and the size - * after decompressing (both with delta bases and zlib). - */ -struct survey_report_object_size_summary { - char *label; - size_t nr; - size_t disk_size; - size_t inflated_size; - size_t num_missing; -}; - -typedef int (*survey_top_cmp)(void *v1, void *v2); - -static int cmp_by_nr(void *v1, void *v2) -{ - struct survey_report_object_size_summary *s1 = v1; - struct survey_report_object_size_summary *s2 = v2; - - if (s1->nr < s2->nr) - return -1; - if (s1->nr > s2->nr) - return 1; - return 0; -} - -static int cmp_by_disk_size(void *v1, void *v2) -{ - struct survey_report_object_size_summary *s1 = v1; - struct survey_report_object_size_summary *s2 = v2; - - if (s1->disk_size < s2->disk_size) - return -1; - if (s1->disk_size > s2->disk_size) - return 1; - return 0; -} - -static int cmp_by_inflated_size(void *v1, void *v2) -{ - struct survey_report_object_size_summary *s1 = v1; - struct survey_report_object_size_summary *s2 = v2; - - if (s1->inflated_size < s2->inflated_size) - return -1; - if (s1->inflated_size > s2->inflated_size) - return 1; - return 0; -} - -/** - * Store a list of "top" categories by some sorting function. When - * inserting a new category, reorder the list and free the one that - * got ejected (if any). - */ -struct survey_report_top_table { - const char *name; - survey_top_cmp cmp_fn; - size_t nr; - size_t alloc; - - /** - * 'data' stores an array of structs and must be cast into - * the proper array type before evaluating an index. - */ - void *data; -}; - -static void init_top_sizes(struct survey_report_top_table *top, - size_t limit, const char *name, - survey_top_cmp cmp) -{ - struct survey_report_object_size_summary *sz_array; - - top->name = name; - top->cmp_fn = cmp; - top->alloc = limit; - top->nr = 0; - - CALLOC_ARRAY(sz_array, limit); - top->data = sz_array; -} - -MAYBE_UNUSED -static void clear_top_sizes(struct survey_report_top_table *top) -{ - struct survey_report_object_size_summary *sz_array = top->data; - - for (size_t i = 0; i < top->nr; i++) - free(sz_array[i].label); - free(sz_array); -} - -static void maybe_insert_into_top_size(struct survey_report_top_table *top, - struct survey_report_object_size_summary *summary) -{ - struct survey_report_object_size_summary *sz_array = top->data; - size_t pos = top->nr; - - /* Compare against list from the bottom. */ - while (pos > 0 && top->cmp_fn(&sz_array[pos - 1], summary) < 0) - pos--; - - /* Not big enough! */ - if (pos >= top->alloc) - return; - - /* We need to shift the data. */ - if (top->nr == top->alloc) - free(sz_array[top->nr - 1].label); - else - top->nr++; - - for (size_t i = top->nr - 1; i > pos; i--) - memcpy(&sz_array[i], &sz_array[i - 1], sizeof(*sz_array)); - - memcpy(&sz_array[pos], summary, sizeof(*summary)); - sz_array[pos].label = xstrdup(summary->label); -} - -/** - * This struct contains all of the information that needs to be printed - * at the end of the exploration of the repository and its references. - */ -struct survey_report { - struct survey_report_ref_summary refs; - struct survey_report_object_summary reachable_objects; - - struct survey_report_object_size_summary *by_type; - - struct survey_report_top_table *top_paths_by_count; - struct survey_report_top_table *top_paths_by_disk; - struct survey_report_top_table *top_paths_by_inflate; -}; - -#define REPORT_TYPE_COMMIT 0 -#define REPORT_TYPE_TREE 1 -#define REPORT_TYPE_BLOB 2 -#define REPORT_TYPE_TAG 3 -#define REPORT_TYPE_COUNT 4 - -struct survey_context { - struct repository *repo; - - /* Options that control what is done. */ - struct survey_opts opts; - - /* Info for output only. */ - struct survey_report report; - - /* - * The rest of the members are about enabling the activity - * of the 'git survey' command, including ref listings, object - * pointers, and progress. - */ - - struct progress *progress; - size_t progress_nr; - size_t progress_total; - - struct strvec refs; - struct ref_array ref_array; -}; - -static void clear_survey_context(struct survey_context *ctx) -{ - ref_array_clear(&ctx->ref_array); - strvec_clear(&ctx->refs); -} - -struct survey_table { - const char *table_name; - struct strvec header; - struct strvec *rows; - size_t rows_nr; - size_t rows_alloc; -}; - -#define SURVEY_TABLE_INIT { \ - .header = STRVEC_INIT, \ -} - -static void clear_table(struct survey_table *table) -{ - strvec_clear(&table->header); - for (size_t i = 0; i < table->rows_nr; i++) - strvec_clear(&table->rows[i]); - free(table->rows); -} - -static void insert_table_rowv(struct survey_table *table, ...) -{ - va_list ap; - char *arg; - ALLOC_GROW(table->rows, table->rows_nr + 1, table->rows_alloc); - - memset(&table->rows[table->rows_nr], 0, sizeof(struct strvec)); - - va_start(ap, table); - while ((arg = va_arg(ap, char *))) - strvec_push(&table->rows[table->rows_nr], arg); - va_end(ap); - - table->rows_nr++; -} - -#define SECTION_SEGMENT "========================================" -#define SECTION_SEGMENT_LEN 40 -static const char *section_line = SECTION_SEGMENT - SECTION_SEGMENT - SECTION_SEGMENT - SECTION_SEGMENT; -static const size_t section_len = 4 * SECTION_SEGMENT_LEN; - -static void print_table_title(const char *name, size_t *widths, size_t nr) -{ - size_t width = 3 * (nr - 1); - size_t min_width = strlen(name); - - for (size_t i = 0; i < nr; i++) - width += widths[i]; - - if (width < min_width) - width = min_width; - - if (width > section_len) - width = section_len; - - printf("\n%s\n%.*s\n", name, (int)width, section_line); -} - -static void print_row_plaintext(struct strvec *row, size_t *widths) -{ - static struct strbuf line = STRBUF_INIT; - strbuf_setlen(&line, 0); - - for (size_t i = 0; i < row->nr; i++) { - const char *str = row->v[i]; - size_t len = strlen(str); - if (i) - strbuf_add(&line, " | ", 3); - strbuf_addchars(&line, ' ', widths[i] - len); - strbuf_add(&line, str, len); - } - printf("%s\n", line.buf); -} - -static void print_divider_plaintext(size_t *widths, size_t nr) -{ - static struct strbuf line = STRBUF_INIT; - strbuf_setlen(&line, 0); - - for (size_t i = 0; i < nr; i++) { - if (i) - strbuf_add(&line, "-+-", 3); - strbuf_addchars(&line, '-', widths[i]); - } - printf("%s\n", line.buf); -} - -static void print_table_plaintext(struct survey_table *table) -{ - size_t *column_widths; - size_t columns_nr = table->header.nr; - CALLOC_ARRAY(column_widths, columns_nr); - - for (size_t i = 0; i < columns_nr; i++) { - column_widths[i] = strlen(table->header.v[i]); - - for (size_t j = 0; j < table->rows_nr; j++) { - size_t rowlen = strlen(table->rows[j].v[i]); - if (column_widths[i] < rowlen) - column_widths[i] = rowlen; - } - } - - print_table_title(table->table_name, column_widths, columns_nr); - print_row_plaintext(&table->header, column_widths); - print_divider_plaintext(column_widths, columns_nr); - - for (size_t j = 0; j < table->rows_nr; j++) - print_row_plaintext(&table->rows[j], column_widths); - - free(column_widths); -} - -static void survey_report_plaintext_refs(struct survey_context *ctx) -{ - struct survey_report_ref_summary *refs = &ctx->report.refs; - struct survey_table table = SURVEY_TABLE_INIT; - - table.table_name = _("REFERENCES SUMMARY"); - - strvec_push(&table.header, _("Ref Type")); - strvec_push(&table.header, _("Count")); - - if (ctx->opts.refs.want_all_refs || ctx->opts.refs.want_branches) { - char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->branches_nr); - insert_table_rowv(&table, _("Branches"), fmt, NULL); - free(fmt); - } - - if (ctx->opts.refs.want_all_refs || ctx->opts.refs.want_remotes) { - char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->remote_refs_nr); - insert_table_rowv(&table, _("Remote refs"), fmt, NULL); - free(fmt); - } - - if (ctx->opts.refs.want_all_refs || ctx->opts.refs.want_tags) { - char *fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->tags_nr); - insert_table_rowv(&table, _("Tags (all)"), fmt, NULL); - free(fmt); - fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)refs->tags_annotated_nr); - insert_table_rowv(&table, _("Tags (annotated)"), fmt, NULL); - free(fmt); - } - - print_table_plaintext(&table); - clear_table(&table); -} - -static void survey_report_plaintext_reachable_object_summary(struct survey_context *ctx) -{ - struct survey_report_object_summary *objs = &ctx->report.reachable_objects; - struct survey_table table = SURVEY_TABLE_INIT; - char *fmt; - - table.table_name = _("REACHABLE OBJECT SUMMARY"); - - strvec_push(&table.header, _("Object Type")); - strvec_push(&table.header, _("Count")); - - fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->tags_nr); - insert_table_rowv(&table, _("Tags"), fmt, NULL); - free(fmt); - - fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->commits_nr); - insert_table_rowv(&table, _("Commits"), fmt, NULL); - free(fmt); - - fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->trees_nr); - insert_table_rowv(&table, _("Trees"), fmt, NULL); - free(fmt); - - fmt = xstrfmt("%"PRIuMAX"", (uintmax_t)objs->blobs_nr); - insert_table_rowv(&table, _("Blobs"), fmt, NULL); - free(fmt); - - print_table_plaintext(&table); - clear_table(&table); -} - -static void survey_report_object_sizes(const char *title, - const char *categories, - struct survey_report_object_size_summary *summary, - size_t summary_nr) -{ - struct survey_table table = SURVEY_TABLE_INIT; - table.table_name = title; - - strvec_push(&table.header, categories); - strvec_push(&table.header, _("Count")); - strvec_push(&table.header, _("Disk Size")); - strvec_push(&table.header, _("Inflated Size")); - - for (size_t i = 0; i < summary_nr; i++) { - char *label_str = xstrdup(summary[i].label); - char *nr_str = xstrfmt("%"PRIuMAX, (uintmax_t)summary[i].nr); - char *disk_str = xstrfmt("%"PRIuMAX, (uintmax_t)summary[i].disk_size); - char *inflate_str = xstrfmt("%"PRIuMAX, (uintmax_t)summary[i].inflated_size); - - insert_table_rowv(&table, label_str, nr_str, - disk_str, inflate_str, NULL); - - free(label_str); - free(nr_str); - free(disk_str); - free(inflate_str); - } - - print_table_plaintext(&table); - clear_table(&table); -} - -static void survey_report_plaintext_sorted_size( - struct survey_report_top_table *top) -{ - survey_report_object_sizes(top->name, _("Path"), - top->data, top->nr); -} - -static void survey_report_plaintext(struct survey_context *ctx) -{ - printf("GIT SURVEY for \"%s\"\n", ctx->repo->worktree); - printf("-----------------------------------------------------\n"); - survey_report_plaintext_refs(ctx); - survey_report_plaintext_reachable_object_summary(ctx); - survey_report_object_sizes(_("TOTAL OBJECT SIZES BY TYPE"), - _("Object Type"), - ctx->report.by_type, - REPORT_TYPE_COUNT); - - survey_report_plaintext_sorted_size( - &ctx->report.top_paths_by_count[REPORT_TYPE_TREE]); - survey_report_plaintext_sorted_size( - &ctx->report.top_paths_by_count[REPORT_TYPE_BLOB]); - - survey_report_plaintext_sorted_size( - &ctx->report.top_paths_by_disk[REPORT_TYPE_TREE]); - survey_report_plaintext_sorted_size( - &ctx->report.top_paths_by_disk[REPORT_TYPE_BLOB]); - - survey_report_plaintext_sorted_size( - &ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE]); - survey_report_plaintext_sorted_size( - &ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB]); -} - /* - * After parsing the command line arguments, figure out which refs we - * should scan. - * - * If ANY were given in positive sense, then we ONLY include them and - * do not use the builtin values. + * `git survey` has been superseded by `git repo structure`. To keep + * older callers working while the migration completes, accept the + * `git survey` command line, translate the options into the + * equivalent `git repo structure` invocation, and re-exec. */ -static void fixup_refs_wanted(struct survey_context *ctx) -{ - struct survey_refs_wanted *rw = &ctx->opts.refs; - - /* - * `--all-refs` overrides and enables everything. - */ - if (rw->want_all_refs == 1) { - rw->want_branches = 1; - rw->want_tags = 1; - rw->want_remotes = 1; - rw->want_detached = 1; - rw->want_other = 1; - return; - } +int cmd_survey(int argc, const char **argv, const char *prefix, + struct repository *repo UNUSED) +{ + int verbose = 0; + int show_progress = -1; + int top_nr = 10; + int want_all_refs = -1; + int want_branches = -1; + int want_tags = -1; + int want_remotes = -1; + int want_detached = -1; + int want_other = -1; + struct strvec child_argv = STRVEC_INIT; + struct option options[] = { + OPT__VERBOSE(&verbose, N_("verbose output (ignored)")), + OPT_BOOL(0, "progress", &show_progress, N_("show progress")), + OPT_INTEGER('n', "top", &top_nr, + N_("number of entries to include in " + "detail tables")), + OPT_BOOL_F(0, "all-refs", &want_all_refs, + N_("include all refs"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "branches", &want_branches, + N_("include branches"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "tags", &want_tags, + N_("include tags"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "remotes", &want_remotes, + N_("include remote refs"), PARSE_OPT_NONEG), + OPT_BOOL_F(0, "detached", &want_detached, + N_("include detached HEAD (ignored)"), + PARSE_OPT_NONEG), + OPT_BOOL_F(0, "other", &want_other, + N_("include notes and stashes"), PARSE_OPT_NONEG), + OPT_END(), + }; - /* - * If none of the `--` were given, we assume all - * of the builtin unspecified values. - */ - if (rw->want_branches == -1 && - rw->want_tags == -1 && - rw->want_remotes == -1 && - rw->want_detached == -1 && - rw->want_other == -1) { - *rw = default_ref_options; - return; - } + argc = parse_options(argc, argv, prefix, options, survey_usage, 0); + if (top_nr < 0) + die(_("--top= must be non-negative")); + if (argc) + usage(_("'git survey' takes no positional arguments")); + + warning(_("'git survey' is deprecated; " + "use 'git repo structure' instead")); + if (verbose) + warning(_("--verbose is ignored by 'git repo structure'")); + if (want_detached != -1) + warning(_("--detached is ignored by 'git repo structure'")); + + strvec_pushl(&child_argv, "repo", "structure", NULL); + if (show_progress == 1) + strvec_push(&child_argv, "--progress"); + else if (show_progress == 0) + strvec_push(&child_argv, "--no-progress"); + strvec_pushf(&child_argv, "--top=%d", top_nr); /* - * Since we only allow positive boolean values on the command - * line, we will only have true values where they specified - * a `--`. - * - * So anything that still has an unspecified value should be - * set to false. + * Survey's default ref scope is branches+tags+remotes (not "other"). + * `--all-refs` widens to literally everything; the per-kind flags + * select specific subsets. `git repo structure` defaults to all + * refs and accepts a repeatable --ref-filter=, so the + * translation is straightforward. */ - if (rw->want_branches == -1) - rw->want_branches = 0; - if (rw->want_tags == -1) - rw->want_tags = 0; - if (rw->want_remotes == -1) - rw->want_remotes = 0; - if (rw->want_detached == -1) - rw->want_detached = 0; - if (rw->want_other == -1) - rw->want_other = 0; -} - -static int survey_load_config_cb(const char *var, const char *value, - const struct config_context *cctx, void *pvoid) -{ - struct survey_context *ctx = pvoid; - - if (!strcmp(var, "survey.verbose")) { - ctx->opts.verbose = git_config_bool(var, value); - return 0; - } - if (!strcmp(var, "survey.progress")) { - ctx->opts.show_progress = git_config_bool(var, value); - return 0; - } - if (!strcmp(var, "survey.top")) { - ctx->opts.top_nr = git_config_bool(var, value); - return 0; - } - - return git_default_config(var, value, cctx, pvoid); -} - -static void survey_load_config(struct survey_context *ctx) -{ - repo_config(the_repository, survey_load_config_cb, ctx); -} - -static void do_load_refs(struct survey_context *ctx, - struct ref_array *ref_array) -{ - struct ref_filter filter = REF_FILTER_INIT; - struct ref_sorting *sorting; - struct string_list sorting_options = STRING_LIST_INIT_DUP; - - string_list_append(&sorting_options, "objectname"); - sorting = ref_sorting_options(&sorting_options); - - if (ctx->opts.refs.want_detached) - strvec_push(&ctx->refs, "HEAD"); - - if (ctx->opts.refs.want_all_refs) { - strvec_push(&ctx->refs, "refs/"); - } else { - if (ctx->opts.refs.want_branches) - strvec_push(&ctx->refs, "refs/heads/"); - if (ctx->opts.refs.want_tags) - strvec_push(&ctx->refs, "refs/tags/"); - if (ctx->opts.refs.want_remotes) - strvec_push(&ctx->refs, "refs/remotes/"); - if (ctx->opts.refs.want_other) { - strvec_push(&ctx->refs, "refs/notes/"); - strvec_push(&ctx->refs, "refs/stash/"); - } - } - - filter.name_patterns = ctx->refs.v; - filter.ignore_case = 0; - filter.match_as_path = 1; - - if (ctx->opts.show_progress) { - ctx->progress_total = 0; - ctx->progress = start_progress(ctx->repo, - _("Scanning refs..."), 0); - } - - filter_refs(ref_array, &filter, FILTER_REFS_KIND_MASK); - - if (ctx->opts.show_progress) { - ctx->progress_total = ref_array->nr; - display_progress(ctx->progress, ctx->progress_total); - } - - ref_array_sort(sorting, ref_array); - - stop_progress(&ctx->progress); - ref_filter_clear(&filter); - ref_sorting_release(sorting); -} - -/* - * The REFS phase: - * - * Load the set of requested refs and assess them for scalablity problems. - * Use that set to start a treewalk to all reachable objects and assess - * them. - * - * This data will give us insights into the repository itself (the number - * of refs, the size and shape of the DAG, the number and size of the - * objects). - * - * Theoretically, this data is independent of the on-disk representation - * (e.g. independent of packing concerns). - */ -static void survey_phase_refs(struct survey_context *ctx) -{ - trace2_region_enter("survey", "phase/refs", ctx->repo); - do_load_refs(ctx, &ctx->ref_array); - - ctx->report.refs.refs_nr = ctx->ref_array.nr; - for (int i = 0; i < ctx->ref_array.nr; i++) { - size_t size; - struct ref_array_item *item = ctx->ref_array.items[i]; - - switch (item->kind) { - case FILTER_REFS_TAGS: - ctx->report.refs.tags_nr++; - if (odb_read_object_info(ctx->repo->objects, - &item->objectname, - &size) == OBJ_TAG) - ctx->report.refs.tags_annotated_nr++; - break; - - case FILTER_REFS_BRANCHES: - ctx->report.refs.branches_nr++; - break; - - case FILTER_REFS_REMOTES: - ctx->report.refs.remote_refs_nr++; - break; - - case FILTER_REFS_OTHERS: - ctx->report.refs.others_nr++; - break; - - default: - ctx->report.refs.unknown_nr++; - break; - } - } - - trace2_region_leave("survey", "phase/refs", ctx->repo); -} - -static void increment_object_counts( - struct survey_report_object_summary *summary, - enum object_type type, - size_t nr) -{ - switch (type) { - case OBJ_COMMIT: - summary->commits_nr += nr; - break; - - case OBJ_TREE: - summary->trees_nr += nr; - break; - - case OBJ_BLOB: - summary->blobs_nr += nr; - break; - - case OBJ_TAG: - summary->tags_nr += nr; - break; - - default: - break; - } -} - -static void increment_totals(struct survey_context *ctx, - struct oid_array *oids, - struct survey_report_object_size_summary *summary) -{ - for (size_t i = 0; i < oids->nr; i++) { - struct object_info oi = OBJECT_INFO_INIT; - unsigned oi_flags = OBJECT_INFO_FOR_PREFETCH; - size_t object_length = 0; - off_t disk_sizep = 0; - enum object_type type; - - oi.typep = &type; - oi.sizep = &object_length; - oi.disk_sizep = &disk_sizep; - - if (odb_read_object_info_extended(ctx->repo->objects, - &oids->oid[i], - &oi, oi_flags) < 0) { - summary->num_missing++; - } else { - summary->nr++; - summary->disk_size += disk_sizep; - summary->inflated_size += object_length; + if (want_all_refs != 1) { + int branches = want_branches == 1; + int tags = want_tags == 1; + int remotes = want_remotes == 1; + int other = want_other == 1; + + if (!branches && !tags && !remotes && !other) + branches = tags = remotes = 1; + + if (branches) + strvec_push(&child_argv, + "--ref-filter=refs/heads/*"); + if (tags) + strvec_push(&child_argv, + "--ref-filter=refs/tags/*"); + if (remotes) + strvec_push(&child_argv, + "--ref-filter=refs/remotes/*"); + if (other) { + strvec_push(&child_argv, + "--ref-filter=refs/notes/*"); + strvec_push(&child_argv, + "--ref-filter=refs/stash"); } } -} - -static void increment_object_totals(struct survey_context *ctx, - struct oid_array *oids, - enum object_type type, - const char *path) -{ - struct survey_report_object_size_summary *total; - struct survey_report_object_size_summary summary = { 0 }; - - increment_totals(ctx, oids, &summary); - - switch (type) { - case OBJ_COMMIT: - total = &ctx->report.by_type[REPORT_TYPE_COMMIT]; - break; - - case OBJ_TREE: - total = &ctx->report.by_type[REPORT_TYPE_TREE]; - break; - - case OBJ_BLOB: - total = &ctx->report.by_type[REPORT_TYPE_BLOB]; - break; - - case OBJ_TAG: - total = &ctx->report.by_type[REPORT_TYPE_TAG]; - break; - - default: - BUG("No other type allowed"); - } - - total->nr += summary.nr; - total->disk_size += summary.disk_size; - total->inflated_size += summary.inflated_size; - total->num_missing += summary.num_missing; - - if (type == OBJ_TREE || type == OBJ_BLOB) { - int index = type == OBJ_TREE ? - REPORT_TYPE_TREE : REPORT_TYPE_BLOB; - struct survey_report_top_table *top; - - /* - * Temporarily store (const char *) here, but it will - * be duped if inserted and will not be freed. - */ - summary.label = (char *)path; - - top = ctx->report.top_paths_by_count; - maybe_insert_into_top_size(&top[index], &summary); - - top = ctx->report.top_paths_by_disk; - maybe_insert_into_top_size(&top[index], &summary); - - top = ctx->report.top_paths_by_inflate; - maybe_insert_into_top_size(&top[index], &summary); - } -} - -static int survey_objects_path_walk_fn(const char *path, - struct oid_array *oids, - enum object_type type, - void *data) -{ - struct survey_context *ctx = data; - - increment_object_counts(&ctx->report.reachable_objects, - type, oids->nr); - increment_object_totals(ctx, oids, type, path); - - ctx->progress_nr += oids->nr; - display_progress(ctx->progress, ctx->progress_nr); - - return 0; -} - -static void initialize_report(struct survey_context *ctx) -{ - CALLOC_ARRAY(ctx->report.by_type, REPORT_TYPE_COUNT); - ctx->report.by_type[REPORT_TYPE_COMMIT].label = xstrdup(_("Commits")); - ctx->report.by_type[REPORT_TYPE_TREE].label = xstrdup(_("Trees")); - ctx->report.by_type[REPORT_TYPE_BLOB].label = xstrdup(_("Blobs")); - ctx->report.by_type[REPORT_TYPE_TAG].label = xstrdup(_("Tags")); - - CALLOC_ARRAY(ctx->report.top_paths_by_count, REPORT_TYPE_COUNT); - init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_TREE], - ctx->opts.top_nr, _("TOP DIRECTORIES BY COUNT"), cmp_by_nr); - init_top_sizes(&ctx->report.top_paths_by_count[REPORT_TYPE_BLOB], - ctx->opts.top_nr, _("TOP FILES BY COUNT"), cmp_by_nr); - - CALLOC_ARRAY(ctx->report.top_paths_by_disk, REPORT_TYPE_COUNT); - init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_TREE], - ctx->opts.top_nr, _("TOP DIRECTORIES BY DISK SIZE"), cmp_by_disk_size); - init_top_sizes(&ctx->report.top_paths_by_disk[REPORT_TYPE_BLOB], - ctx->opts.top_nr, _("TOP FILES BY DISK SIZE"), cmp_by_disk_size); - - CALLOC_ARRAY(ctx->report.top_paths_by_inflate, REPORT_TYPE_COUNT); - init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_TREE], - ctx->opts.top_nr, _("TOP DIRECTORIES BY INFLATED SIZE"), cmp_by_inflated_size); - init_top_sizes(&ctx->report.top_paths_by_inflate[REPORT_TYPE_BLOB], - ctx->opts.top_nr, _("TOP FILES BY INFLATED SIZE"), cmp_by_inflated_size); -} - -static void survey_phase_objects(struct survey_context *ctx) -{ - struct rev_info revs = REV_INFO_INIT; - struct path_walk_info info = PATH_WALK_INFO_INIT; - unsigned int add_flags = 0; - - trace2_region_enter("survey", "phase/objects", ctx->repo); - - info.revs = &revs; - info.path_fn = survey_objects_path_walk_fn; - info.path_fn_data = ctx; - - initialize_report(ctx); - - repo_init_revisions(ctx->repo, &revs, ""); - revs.tag_objects = 1; - - ctx->progress_nr = 0; - ctx->progress_total = ctx->ref_array.nr; - if (ctx->opts.show_progress) - ctx->progress = start_progress(ctx->repo, - _("Preparing object walk"), - ctx->progress_total); - for (int i = 0; i < ctx->ref_array.nr; i++) { - struct ref_array_item *item = ctx->ref_array.items[i]; - add_pending_oid(&revs, NULL, &item->objectname, add_flags); - display_progress(ctx->progress, ++(ctx->progress_nr)); - } - stop_progress(&ctx->progress); - - ctx->progress_nr = 0; - ctx->progress_total = 0; - if (ctx->opts.show_progress) - ctx->progress = start_progress(ctx->repo, - _("Walking objects"), 0); - walk_objects_by_path(&info); - stop_progress(&ctx->progress); - - release_revisions(&revs); - trace2_region_leave("survey", "phase/objects", ctx->repo); -} - -int cmd_survey(int argc, const char **argv, const char *prefix, struct repository *repo) -{ - static struct survey_context ctx = { - .opts = { - .verbose = 0, - .show_progress = -1, /* defaults to isatty(2) */ - .top_nr = 10, - - .refs.want_all_refs = -1, - - .refs.want_branches = -1, /* default these to undefined */ - .refs.want_tags = -1, - .refs.want_remotes = -1, - .refs.want_detached = -1, - .refs.want_other = -1, - }, - .refs = STRVEC_INIT, - }; - - static struct option survey_options[] = { - OPT__VERBOSE(&ctx.opts.verbose, N_("verbose output")), - OPT_BOOL(0, "progress", &ctx.opts.show_progress, N_("show progress")), - OPT_INTEGER('n', "top", &ctx.opts.top_nr, - N_("number of entries to include in detail tables")), - - OPT_BOOL_F(0, "all-refs", &ctx.opts.refs.want_all_refs, N_("include all refs"), PARSE_OPT_NONEG), - - OPT_BOOL_F(0, "branches", &ctx.opts.refs.want_branches, N_("include branches"), PARSE_OPT_NONEG), - OPT_BOOL_F(0, "tags", &ctx.opts.refs.want_tags, N_("include tags"), PARSE_OPT_NONEG), - OPT_BOOL_F(0, "remotes", &ctx.opts.refs.want_remotes, N_("include all remotes refs"), PARSE_OPT_NONEG), - OPT_BOOL_F(0, "detached", &ctx.opts.refs.want_detached, N_("include detached HEAD"), PARSE_OPT_NONEG), - OPT_BOOL_F(0, "other", &ctx.opts.refs.want_other, N_("include notes and stashes"), PARSE_OPT_NONEG), - - OPT_END(), - }; - - show_usage_with_options_if_asked(argc, argv, - survey_usage, survey_options); - - if (isatty(2)) - color_fprintf_ln(stderr, - want_color_fd(2, GIT_COLOR_AUTO) ? GIT_COLOR_YELLOW : "", - "(THIS IS EXPERIMENTAL, EXPECT THE OUTPUT FORMAT TO CHANGE!)"); - - ctx.repo = repo; - - prepare_repo_settings(ctx.repo); - survey_load_config(&ctx); - - argc = parse_options(argc, argv, prefix, survey_options, survey_usage, 0); - - if (ctx.opts.show_progress < 0) - ctx.opts.show_progress = isatty(2); - - fixup_refs_wanted(&ctx); - - survey_phase_refs(&ctx); - - survey_phase_objects(&ctx); - - survey_report_plaintext(&ctx); - clear_survey_context(&ctx); - return 0; + execv_git_cmd(child_argv.v); + /* unreachable: execv_git_cmd dies on failure */ + strvec_clear(&child_argv); + return 1; } diff --git a/t/t8100-git-survey.sh b/t/t8100-git-survey.sh index 703860d22ff4c3..a69d8f26724b9d 100755 --- a/t/t8100-git-survey.sh +++ b/t/t8100-git-survey.sh @@ -1,6 +1,6 @@ #!/bin/sh -test_description='git survey' +test_description='git survey (deprecated shim over `git repo structure`)' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME @@ -10,9 +10,9 @@ export TEST_PASSES_SANITIZE_LEAK . ./test-lib.sh -test_expect_success 'git survey -h shows experimental warning' ' +test_expect_success 'git survey -h shows the deprecated banner' ' test_expect_code 0 git survey -h >usage && - test_grep "EXPERIMENTAL!" usage + test_grep "DEPRECATED!" usage ' test_expect_success 'create a semi-interesting repo' ' @@ -25,84 +25,28 @@ test_expect_success 'create a semi-interesting repo' ' git update-ref -d refs/tags/two ' -test_expect_success 'git survey --progress' ' - GIT_PROGRESS_DELAY=0 git survey --all-refs --progress >out 2>err && - grep "Preparing object walk" err -' - -approximate_sizes() { - # very simplistic approximate rounding - sed -Ee "s/ *(1[0-9][0-9])( |$)/ ~0.1kB\2/g" \ - -e "s/ *(4[6-9][0-9]|5[0-6][0-9])( |$)/ ~0.5kB\2/g" \ - -e "s/ *(5[6-9][0-9]|6[0-6][0-9])( |$)/ ~0.6kB\2/g" \ - -e "s/ *1(4[89][0-9]|5[0-8][0-9])( |$)/ ~1.5kB\2/g" \ - -e "s/ *1(69[0-9]|7[0-9][0-9])( |$)/ ~1.7kB\2/g" \ - -e "s/ *1(79[0-9]|8[0-9][0-9])( |$)/ ~1.8kB\2/g" \ - -e "s/ *2(1[0-9][0-9]|20[0-1])( |$)/ ~2.1kB\2/g" \ - -e "s/ *2(3[0-9][0-9]|4[0-1][0-9])( |$)/ ~2.3kB\2/g" \ - -e "s/ *2(5[0-9][0-9]|6[0-1][0-9])( |$)/ ~2.5kB\2/g" \ - "$@" -} - -test_expect_success 'git survey (default)' ' +test_expect_success 'survey prints a deprecation warning' ' git survey --all-refs >out 2>err && - test_line_count = 0 err && - - test_oid_cache <<-EOF && - commits_sizes sha1:~1.5kB | ~2.1kB - commits_sizes sha256:~1.8kB | ~2.5kB - trees_sizes sha1:~0.5kB | ~1.7kB - trees_sizes sha256:~0.6kB | ~2.3kB - blobs_sizes sha1:~0.1kB | ~0.1kB - blobs_sizes sha256:~0.1kB | ~0.1kB - tags_sizes sha1:~0.5kB | ~0.5kB - tags_sizes sha256:~0.5kB | ~0.6kB - EOF - - tr , " " >expect <<-EOF && - GIT SURVEY for "$(pwd)" - ----------------------------------------------------- - - REFERENCES SUMMARY - ======================== - , Ref Type | Count - -----------------+------ - , Branches | 1 - Remote refs | 0 - Tags (all) | 2 - Tags (annotated) | 2 - - REACHABLE OBJECT SUMMARY - ======================== - Object Type | Count - ------------+------ - Tags | 4 - Commits | 10 - Trees | 10 - Blobs | 10 + test_grep "is deprecated" err +' - TOTAL OBJECT SIZES BY TYPE - =============================================== - Object Type | Count | Disk Size | Inflated Size - ------------+-------+-----------+-------------- - Commits | 10 | $(test_oid commits_sizes) - Trees | 10 | $(test_oid trees_sizes) - Blobs | 10 | $(test_oid blobs_sizes) - Tags | 4 | $(test_oid tags_sizes) - EOF +test_expect_success 'survey forwards to git repo structure' ' + git survey --all-refs >survey-out 2>survey-err && + git repo structure --top=10 >structure-out 2>structure-err && + test_cmp structure-out survey-out +' - approximate_sizes out >out-edited && - lines=$(wc -l out-trimmed && - test_cmp expect out-trimmed && +test_expect_success 'survey --top is translated' ' + git survey --top=3 --all-refs >out && + git repo structure --top=3 >expected && + test_cmp expected out +' - for type in "DIRECTORIES" "FILES" - do - for metric in "COUNT" "DISK SIZE" "INFLATED SIZE" - do - grep "TOP $type BY $metric" out || return 1 - done || return 1 - done +test_expect_success 'survey --branches translates to a refs/heads/* filter' ' + git survey --branches >out && + git repo structure --top=10 \ + --ref-filter="refs/heads/*" >expected && + test_cmp expected out ' test_done From 6cea43899108a78ba392fea184a5683a2f31a036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20A=C3=9Fhauer?= Date: Sun, 22 Dec 2024 17:24:24 +0100 Subject: [PATCH 115/229] credential-cache: handle ECONNREFUSED gracefully MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In 245670c (credential-cache: check for windows specific errors, 2021-09-14) we concluded that on Windows we would always encounter ENETDOWN where we would expect ECONNREFUSED on POSIX systems, when connecting to unix sockets. As reported in [1], we do encounter ECONNREFUSED on Windows if the socket file doesn't exist, but the containing directory does and ENETDOWN if neither exists. We should handle this case like we do on non-windows systems. [1] https://github.com/git-for-windows/git/pull/4762#issuecomment-2545498245 This fixes https://github.com/git-for-windows/git/issues/5314 Helped-by: M Hickford Signed-off-by: Matthias Aßhauer Signed-off-by: Johannes Schindelin --- builtin/credential-cache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/builtin/credential-cache.c b/builtin/credential-cache.c index 7f733cb756e03c..3b8130d3d64f9c 100644 --- a/builtin/credential-cache.c +++ b/builtin/credential-cache.c @@ -23,7 +23,7 @@ static int connection_closed(int error) static int connection_fatally_broken(int error) { - return (error != ENOENT) && (error != ENETDOWN); + return (error != ENOENT) && (error != ENETDOWN) && (error != ECONNREFUSED); } #else From 8c422b66d2db5af5b9f89a57cd6affa87d6f00c5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 6 Mar 2025 14:05:03 +0100 Subject: [PATCH 116/229] reftable: do make sure to use custom allocators The reftable library goes out of its way to use its own set of allocator functions that can be configured using `reftable_set_alloc()`. However, Git does not configure this. That is not typically a problem, except when Git uses a custom allocator via some definitions in `git-compat-util.h`, as is the case in Git for Windows (which switched away from the long-unmaintained nedmalloc to mimalloc). Then, it is quite possible that Git assigns a `strbuf` (allocated via the custom allocator) to, say, the `refname` field of a `reftable_log_record` in `write_transaction_table()`, and later on asks the reftable library function `reftable_log_record_release()` to release it, but that function was compiled without using `git-compat-util.h` and hence calls regular `free()` (i.e. _not_ the custom allocator's own function). This has been a problem for a long time and it was a matter of some sort of "luck" that 1) reftables are not commonly used on Windows, and 2) mimalloc can often ignore gracefully when it is asked to release memory that it has not allocated. However, a recent update to `seen` brought this problem to the forefront, letting t1460 fail in Git for Windows, with symptoms much in the same way as the problem I had to address in d02c37c3e6ba (t-reftable-basics: allow for `malloc` to be `#define`d, 2025-01-08) where exit code 127 was also produced in lieu of `STATUS_HEAP_CORRUPTION` (C0000374) because exit codes are only 7 bits wide. It was not possible to figure out what change in particular caused these new failures within a reasonable time frame, as there are too many changes in `seen` that conflict with Git for Windows' patches, I had to stop the investigation after spending four hours on it fruitlessly. To verify that this patch fixes the issue, I avoided using mimalloc and temporarily patched in a "custom allocator" that would more reliably point out problems, like this: diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index 68f38291f84c..9421d630b9f5 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -353,6 +353,69 @@ static int reftable_be_fsync(int fd) return fsync_component(FSYNC_COMPONENT_REFERENCE, fd); } +#define DEBUG_REFTABLE_ALLOC +#ifdef DEBUG_REFTABLE_ALLOC +#include "khash.h" + +static inline khint_t __ac_X31_hash_ptr(void *ptr) +{ + union { + void *ptr; + char s[sizeof(void *)]; + } u; + size_t i; + khint_t h; + + u.ptr = ptr; + h = (khint_t)*u.s; + for (i = 0; i < sizeof(void *); i++) + h = (h << 5) - h + (khint_t)u.s[i]; + return h; +} + +#define kh_ptr_hash_func(key) __ac_X31_hash_ptr(key) +#define kh_ptr_hash_equal(a, b) ((a) == (b)) + +KHASH_INIT(ptr, void *, int, 0, kh_ptr_hash_func, kh_ptr_hash_equal) + +static kh_ptr_t *my_malloced; + +static void *my_malloc(size_t sz) +{ + int dummy; + void *ptr = malloc(sz); + if (ptr) + kh_put_ptr(my_malloced, ptr, &dummy); + return ptr; +} + +static void *my_realloc(void *ptr, size_t sz) +{ + int dummy; + if (ptr) { + khiter_t pos = kh_get_ptr(my_malloced, ptr); + if (pos >= kh_end(my_malloced)) + die("Was not my_malloc()ed: %p", ptr); + kh_del_ptr(my_malloced, pos); + } + ptr = realloc(ptr, sz); + if (ptr) + kh_put_ptr(my_malloced, ptr, &dummy); + return ptr; +} + +static void my_free(void *ptr) +{ + if (ptr) { + khiter_t pos = kh_get_ptr(my_malloced, ptr); + if (pos >= kh_end(my_malloced)) + die("Was not my_malloc()ed: %p", ptr); + kh_del_ptr(my_malloced, pos); + } + free(ptr); +} +#endif + static struct ref_store *reftable_be_init(struct repository *repo, const char *gitdir, unsigned int store_flags) @@ -362,6 +425,11 @@ static struct ref_store *reftable_be_init(struct repository *repo, int is_worktree; mode_t mask; +#ifdef DEBUG_REFTABLE_ALLOC + my_malloced = kh_init_ptr(); + reftable_set_alloc(my_malloc, my_realloc, my_free); +#endif + mask = umask(0); umask(mask); I briefly considered contributing this "custom allocator" patch, too, but it is unwieldy (for example, it would not work at all when compiling with mimalloc support) and it would only waste space (or even time, if a compile flag was introduced and exercised as part of the CI builds). Given that it is highly unlikely that Git will lose the new `reftable_set_alloc()` call by mistake, I rejected that idea as simply too wasteful. Signed-off-by: Johannes Schindelin --- refs/reftable-backend.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/refs/reftable-backend.c b/refs/reftable-backend.c index d8ebaba8b16db7..286730d28e1d15 100644 --- a/refs/reftable-backend.c +++ b/refs/reftable-backend.c @@ -415,6 +415,8 @@ static struct ref_store *reftable_be_init(struct repository *repo, struct strbuf path = STRBUF_INIT; bool is_worktree; + reftable_set_alloc(malloc, realloc, free); + refs_compute_filesystem_location(gitdir, payload, &is_worktree, &refdir, &ref_common_dir); From 6d0a19fc983ecfcdc50cdc03b82650bf2cf99b51 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 3 Jun 2025 12:45:39 +0200 Subject: [PATCH 117/229] check-whitespace: avoid alerts about upstream commits Every once in a while, whitespace errors are introduced in Git for Windows' rebases to newer Git versions, simply by virtue of integrating upstream commits that do not follow upstream Git's own whitespace rule. In Git v2.50.0-rc0, for example, 03f2915541a4 (xdiff: disable cleanup_records heuristic with --minimal, 2025-04-29) introduced a trailing space. Arguably, non-actionable alerts are worse than no alerts at all, so let's suppress those alerts that we cannot do anything about, anyway. Signed-off-by: Johannes Schindelin --- ci/check-whitespace.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ci/check-whitespace.sh b/ci/check-whitespace.sh index c40804394cb079..e590ac0dfd765e 100755 --- a/ci/check-whitespace.sh +++ b/ci/check-whitespace.sh @@ -19,6 +19,7 @@ problems=() commit= commitText= commitTextmd= +committerEmail= goodParent= if ! git rev-parse --quiet --verify "${baseCommit}" @@ -27,7 +28,7 @@ then exit 1 fi -while read dash sha etc +while read dash email sha etc do case "${dash}" in "---") # Line contains commit information. @@ -40,10 +41,14 @@ do commit="${sha}" commitText="${sha} ${etc}" commitTextmd="[${sha}](${url}/commit/${sha}) ${etc}" + committerEmail="${email}" ;; "") ;; *) # Line contains whitespace error information for current commit. + # Quod licet Iovi non licet bovi + test gitster@pobox.com != "$committerEmail" || break + if test -n "${goodParent}" then problems+=("1) --- ${commitTextmd}") @@ -64,7 +69,7 @@ do echo "${dash} ${sha} ${etc}" ;; esac -done <<< "$(git log --check --pretty=format:"---% h% s" "${baseCommit}"..)" +done <<< "$(git log --check --pretty=format:"---% ce% h% s" "${baseCommit}"..)" if test ${#problems[*]} -gt 0 then From e30999c85a8befdd3db9750dd4a951ca40a0c9fa Mon Sep 17 00:00:00 2001 From: Thomas Braun Date: Mon, 26 Jan 2026 19:02:44 +0100 Subject: [PATCH 118/229] t/t5571-prep-push-hook.sh: Add test with writing to stderr The 2.53.0.rc0.windows release candidate had a regression where writing to stderr from a pre-push hook would error out. The regression was fixed in 2.53.0.rc1.windows and the test here ensures that this stays fixed. Signed-off-by: Thomas Braun --- t/t5571-pre-push-hook.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/t/t5571-pre-push-hook.sh b/t/t5571-pre-push-hook.sh index a11b20e378223e..25b8d50c9428a7 100755 --- a/t/t5571-pre-push-hook.sh +++ b/t/t5571-pre-push-hook.sh @@ -138,4 +138,16 @@ test_expect_success 'sigpipe does not cause pre-push hook failure' ' git push parent1 "refs/heads/b/*:refs/heads/b/*" ' +test_expect_success 'can write to stderr' ' + test_hook --clobber pre-push <<-\EOF && + echo foo >/dev/stderr && + exit 0 + EOF + + test_commit third && + echo foo >expect && + git push --quiet parent1 HEAD 2>actual && + test_cmp expect actual +' + test_done From 99b583b75b55de8ed3e5a93dd75e44a5a4a82a87 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 9 Feb 2026 18:21:48 +0100 Subject: [PATCH 119/229] credential: advertise NTLM suppression and allow helpers to re-enable The previous commits disabled NTLM authentication by default due to its cryptographic weaknesses. Users can re-enable it via the config setting http..allowNTLMAuth, but this requires manual intervention. Credential helpers may have knowledge about which servers are trusted for NTLM authentication (e.g., known on-prem Azure DevOps instances). To allow them to signal this trust, introduce a simple negotiation: when NTLM is suppressed and the server offered it, Git advertises ntlm=suppressed to the credential helper. The helper can respond with ntlm=allow to re-enable NTLM for this request. This happens precisely at the point where we would otherwise warn the user about NTLM being suppressed, ensuring the capability is only advertised when relevant. Helped-by: Matthew John Cheetham Signed-off-by: Johannes Schindelin --- credential.c | 5 +++++ credential.h | 3 +++ http.c | 17 +++++++++++++++-- t/t5563-simple-http-auth.sh | 13 ++++++++++++- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/credential.c b/credential.c index 035399d7ee5abd..8babbeabe82d70 100644 --- a/credential.c +++ b/credential.c @@ -360,6 +360,9 @@ int credential_read(struct credential *c, FILE *fp, credential_set_capability(&c->capa_authtype, op_type); else if (!strcmp(value, "state")) credential_set_capability(&c->capa_state, op_type); + } else if (!strcmp(key, "ntlm")) { + if (!strcmp(value, "allow")) + c->ntlm_allow = 1; } else if (!strcmp(key, "continue")) { c->multistage = !!git_config_bool("continue", value); } else if (!strcmp(key, "password_expiry_utc")) { @@ -420,6 +423,8 @@ void credential_write(const struct credential *c, FILE *fp, if (c->ephemeral) credential_write_item(c, fp, "ephemeral", "1", 0); } + if (c->ntlm_suppressed) + credential_write_item(c, fp, "ntlm", "suppressed", 0); credential_write_item(c, fp, "protocol", c->protocol, 1); credential_write_item(c, fp, "host", c->host, 1); credential_write_item(c, fp, "path", c->path, 0); diff --git a/credential.h b/credential.h index b90f666e33d4e2..ee673c2c2260be 100644 --- a/credential.h +++ b/credential.h @@ -177,6 +177,9 @@ struct credential { struct credential_capability capa_authtype; struct credential_capability capa_state; + unsigned ntlm_suppressed:1, + ntlm_allow:1; + char *username; char *password; char *credential; diff --git a/http.c b/http.c index 5742a17a07ad19..2c3cffc26f39da 100644 --- a/http.c +++ b/http.c @@ -666,6 +666,11 @@ static void init_curl_http_auth(CURL *result) credential_fill(the_repository, &http_auth, 1); + if (http_auth.ntlm_allow && !(http_auth_methods & CURLAUTH_NTLM)) { + http_auth_methods |= CURLAUTH_NTLM; + curl_easy_setopt(result, CURLOPT_HTTPAUTH, http_auth_methods); + } + if (http_auth.password) { if (always_auth_proactively()) { /* @@ -1969,6 +1974,8 @@ static int handle_curl_result(struct slot_results *results) } else if (missing_target(results)) return HTTP_MISSING_TARGET; else if (results->http_code == 401) { + http_auth.ntlm_suppressed = (results->auth_avail & CURLAUTH_NTLM) && + !(http_auth_any & CURLAUTH_NTLM); if ((http_auth.username && http_auth.password) ||\ (http_auth.authtype && http_auth.credential)) { if (http_auth.multistage) { @@ -1978,8 +1985,7 @@ static int handle_curl_result(struct slot_results *results) credential_reject(the_repository, &http_auth); if (always_auth_proactively()) http_proactive_auth = PROACTIVE_AUTH_NONE; - if ((results->auth_avail & CURLAUTH_NTLM) && - !(http_auth_any & CURLAUTH_NTLM)) { + if (http_auth.ntlm_suppressed) { warning(_("Due to its cryptographic weaknesses, " "NTLM authentication has been\n" "disabled in Git by default. You can " @@ -2559,6 +2565,13 @@ static int http_request_recoverable(const char *url, http_reauth_prepare(1); } + /* + * Re-enable NTLM auth if the helper allows it and we would + * otherwise suppress authentication via NTLM. + */ + if (http_auth.ntlm_suppressed && http_auth.ntlm_allow) + http_auth_methods |= CURLAUTH_NTLM; + ret = http_request(url, result, target, options); } if (ret == HTTP_RATE_LIMITED) { diff --git a/t/t5563-simple-http-auth.sh b/t/t5563-simple-http-auth.sh index bd3b697cfd9050..434cf46cd7baa1 100755 --- a/t/t5563-simple-http-auth.sh +++ b/t/t5563-simple-http-auth.sh @@ -778,8 +778,19 @@ test_expect_success NTLM 'access using NTLM auth' ' test_must_fail env GIT_TRACE_CURL=1 git \ ls-remote "$HTTPD_URL/ntlm_auth/repo.git" 2>err && test_grep "allowNTLMAuth" err && + + # Can be enabled via config GIT_TRACE_CURL=1 git -c http.$HTTPD_URL.allowNTLMAuth=true \ - ls-remote "$HTTPD_URL/ntlm_auth/repo.git" + ls-remote "$HTTPD_URL/ntlm_auth/repo.git" && + + # Or via credential helper responding with ntlm=allow + set_credential_reply get <<-EOF && + username=user + password=pwd + ntlm=allow + EOF + + git ls-remote "$HTTPD_URL/ntlm_auth/repo.git" ' test_done From a74fd94b36aaf9e8eb9c97ad29614b0c6ef35aba Mon Sep 17 00:00:00 2001 From: Maks Kuznia Date: Mon, 30 Mar 2026 23:18:31 +0200 Subject: [PATCH 120/229] dir: do not traverse mount points It was already decided in ef22148 (clean: do not traverse mount points, 2018-12-07) that we shouldn't traverse NTFS junctions/bind mounts when using `git clean`, partly because they're sometimes used in worktrees. But the same check wasn't applied to `remove_dir_recurse()` in `dir.c`, which `git worktree remove` uses. So removing a worktree suffers the same problem we had previously with `git clean`. Let's add the same guard from ef22148. Signed-off-by: Maks Kuznia --- dir.c | 7 +++++++ t/t2403-worktree-move.sh | 9 +++++++++ 2 files changed, 16 insertions(+) diff --git a/dir.c b/dir.c index f85c7c587519f5..b45bef44469421 100644 --- a/dir.c +++ b/dir.c @@ -3444,6 +3444,13 @@ static int remove_dir_recurse(struct strbuf *path, int flag, int *kept_up) return 0; } + if (is_mount_point(path)) { + /* Do not descend and nuke a mount point or junction. */ + if (kept_up) + *kept_up = 1; + return 0; + } + flag &= ~REMOVE_DIR_KEEP_TOPLEVEL; dir = opendir(path->buf); if (!dir) { diff --git a/t/t2403-worktree-move.sh b/t/t2403-worktree-move.sh index 69768c120773f4..05a38ecba00b86 100755 --- a/t/t2403-worktree-move.sh +++ b/t/t2403-worktree-move.sh @@ -271,4 +271,13 @@ test_expect_success 'move worktree with relative path to absolute path' ' test_cmp expect .git/worktrees/absolute/gitdir ' +test_expect_success MINGW 'worktree remove does not traverse mount points' ' + mkdir target && + >target/dont-remove-me && + git worktree add --detach wt-junction && + cmd //c "mklink /j wt-junction\\mnt target" && + git worktree remove --force wt-junction && + test_path_is_file target/dont-remove-me +' + test_done From 790a855580175fed7fda5bb24adc8e67b2bd576e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20A=C3=9Fhauer?= Date: Sat, 21 Feb 2026 12:11:02 +0100 Subject: [PATCH 121/229] win32: thread-utils: handle multi-socket systems MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While the currently used way to detect the number of CPU cores on Windows is nice and straight-forward, GetSystemInfo() only gives us access to the number of processors within the current group. [1] While that is usually fine for systems with a single physical CPU, separate physical sockets are typically separate groups. Switch to using GetLogicalProcessorInformationEx() to handle multi-socket systems better. [1] https://learn.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-system_info#members This fixes https://github.com/git-for-windows/git/issues/4766 Co-Authored-by: Herman Semenov Signed-off-by: Matthias Aßhauer --- thread-utils.c | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/thread-utils.c b/thread-utils.c index 374890e6b05b69..00e7e9192b3e0b 100644 --- a/thread-utils.c +++ b/thread-utils.c @@ -28,11 +28,28 @@ int online_cpus(void) #endif #ifdef GIT_WINDOWS_NATIVE - SYSTEM_INFO info; - GetSystemInfo(&info); - - if ((int)info.dwNumberOfProcessors > 0) - return (int)info.dwNumberOfProcessors; + DWORD len = 0; + if (!GetLogicalProcessorInformationEx(RelationProcessorCore, NULL, &len) && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + uint8_t *buf = malloc(len); + if (buf) { + if (GetLogicalProcessorInformationEx(RelationProcessorCore, (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) buf, &len)) { + DWORD offset = 0; + int n_cores = 0; + while (offset < len) { + PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX info = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX) (buf + offset); + offset += info->Size; + /* The threads within a core always share a single group. We need to count the bits in the mask to get a thread count. */ + for (KAFFINITY mask = info->Processor.GroupMask[0].Mask; mask; mask >>= 1) + n_cores += mask &1; + } + if (n_cores) { + free(buf); + return n_cores; + } + } + free(buf); + } + } #elif defined(hpux) || defined(__hpux) || defined(_hpux) struct pst_dynamic psd; From b85fb987eddeadfc8c6d3a3d863363b4d43f4bdc Mon Sep 17 00:00:00 2001 From: Matthew John Cheetham Date: Mon, 13 Apr 2026 12:57:12 +0100 Subject: [PATCH 122/229] t5563: add tests for http.emptyAuth with Negotiate Add tests exercising the interaction between http.emptyAuth and servers that advertise Negotiate (SPNEGO) authentication. Verify that auto mode gives Negotiate a chance via empty auth (resulting in two 401 responses before falling through to credential_fill with Basic credentials), and that false mode strips Negotiate immediately (only one 401 response). Signed-off-by: Matthew John Cheetham --- t/t5563-simple-http-auth.sh | 74 +++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/t/t5563-simple-http-auth.sh b/t/t5563-simple-http-auth.sh index 434cf46cd7baa1..085d6b597f0f24 100755 --- a/t/t5563-simple-http-auth.sh +++ b/t/t5563-simple-http-auth.sh @@ -793,4 +793,78 @@ test_expect_success NTLM 'access using NTLM auth' ' git ls-remote "$HTTPD_URL/ntlm_auth/repo.git" ' +test_lazy_prereq SPNEGO 'curl --version | grep -qi "SPNEGO\|GSS-API\|Kerberos\|negotiate"' + +test_expect_success SPNEGO 'http.emptyAuth=auto attempts Negotiate before credential_fill' ' + test_when_finished "per_test_cleanup" && + + set_credential_reply get <<-EOF && + username=alice + password=secret-passwd + EOF + + # Basic base64(alice:secret-passwd) + cat >"$HTTPD_ROOT_PATH/custom-auth.valid" <<-EOF && + id=1 creds=Basic YWxpY2U6c2VjcmV0LXBhc3N3ZA== + EOF + + cat >"$HTTPD_ROOT_PATH/custom-auth.challenge" <<-EOF && + id=1 status=200 + id=default response=WWW-Authenticate: Negotiate + id=default response=WWW-Authenticate: Basic realm="example.com" + EOF + + test_config_global credential.helper test-helper && + GIT_TRACE_CURL="$TRASH_DIRECTORY/trace-auto" \ + git -c http.emptyAuth=auto \ + ls-remote "$HTTPD_URL/custom_auth/repo.git" && + + # In auto mode with a Negotiate+Basic server, there should be + # three 401 responses: (1) initial no-auth request, (2) empty-auth + # retry where Negotiate fails (no Kerberos ticket), (3) libcurl + # internal Negotiate retry. The fourth attempt uses Basic + # credentials from credential_fill and succeeds. + grep "HTTP/[0-9.]* 401" "$TRASH_DIRECTORY/trace-auto" >actual_401s && + test_line_count = 3 actual_401s && + + expect_credential_query get <<-EOF + capability[]=authtype + capability[]=state + protocol=http + host=$HTTPD_DEST + wwwauth[]=Negotiate + wwwauth[]=Basic realm="example.com" + EOF +' + +test_expect_success SPNEGO 'http.emptyAuth=false skips Negotiate' ' + test_when_finished "per_test_cleanup" && + + set_credential_reply get <<-EOF && + username=alice + password=secret-passwd + EOF + + # Basic base64(alice:secret-passwd) + cat >"$HTTPD_ROOT_PATH/custom-auth.valid" <<-EOF && + id=1 creds=Basic YWxpY2U6c2VjcmV0LXBhc3N3ZA== + EOF + + cat >"$HTTPD_ROOT_PATH/custom-auth.challenge" <<-EOF && + id=1 status=200 + id=default response=WWW-Authenticate: Negotiate + id=default response=WWW-Authenticate: Basic realm="example.com" + EOF + + test_config_global credential.helper test-helper && + GIT_TRACE_CURL="$TRASH_DIRECTORY/trace-false" \ + git -c http.emptyAuth=false \ + ls-remote "$HTTPD_URL/custom_auth/repo.git" && + + # With emptyAuth=false, Negotiate is stripped immediately and + # credential_fill is called right away. Only one 401 response. + grep "HTTP/[0-9.]* 401" "$TRASH_DIRECTORY/trace-false" >actual_401s && + test_line_count = 1 actual_401s +' + test_done From 55bc8a3b6cd1038b611308327b5d3ec81c1014f9 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Mon, 18 May 2026 09:50:39 -0700 Subject: [PATCH 123/229] entry: flush fscache after creating directories and writing files When checkout.workers > 1 and core.fscache is enabled on Windows, 'git checkout -- ' fails when restoring files into directories that do not yet exist on disk. Two failure modes occur: 1. create_directories(): the fscache returns a stale directory listing that does not include a just-created directory. has_dirs_only_path() reports it as non-existent, triggering the unlink+mkdir recovery path which fails with 'cannot create directory: Directory not empty'. 2. write_pc_item(): after writing and closing a file, lstat() cannot see it through the stale fscache, failing with 'unable to stat just-written file'. With workers=1, write_entry() calls flush_fscache() after each file, keeping the cache in sync. With workers>1, enqueue_checkout() defers the write (and the flush), leaving the cache stale for subsequent entries. Fix both by adding flush_fscache() calls after mkdir() in create_directories() and before lstat() in write_pc_item(). On non-Windows platforms flush_fscache() is a no-op. Assisted-by: Claude Opus 4.6 Signed-off-by: Tyrie Vella --- entry.c | 15 +++++++++- parallel-checkout.c | 7 +++++ t/t2080-parallel-checkout-basics.sh | 46 +++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/entry.c b/entry.c index 1c4f0f44070ea3..064e13437235b4 100644 --- a/entry.c +++ b/entry.c @@ -49,10 +49,23 @@ static void create_directories(const char *path, int path_len, */ if (mkdir(buf, 0777)) { if (errno == EEXIST && state->force && - !unlink_or_warn(buf) && !mkdir(buf, 0777)) + !unlink_or_warn(buf) && !mkdir(buf, 0777)) { + flush_fscache(); continue; + } die_errno("cannot create directory at '%s'", buf); } + + /* + * Flush the lstat cache of directory listings so that + * subsequent has_dirs_only_path() calls see the + * just-created directory. Without this, the Windows + * fscache returns stale ENOENT for the new directory, + * causing the next entry sharing this parent to + * incorrectly hit the mkdir/unlink recovery path + * above, which then fails with "Directory not empty". + */ + flush_fscache(); } free(buf); } diff --git a/parallel-checkout.c b/parallel-checkout.c index 0bf4bd6d4abd8c..a6d07dcb1805b9 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -395,6 +395,13 @@ void write_pc_item(struct parallel_checkout_item *pc_item, goto out; } + /* + * Flush the Windows fscache so that the lstat() below sees the + * file we just wrote. Without this, the cached parent directory + * listing may not yet include the new file entry. + */ + flush_fscache(); + if (state->refresh_cache && !fstat_done && lstat(path.buf, &pc_item->st) < 0) { error_errno("unable to stat just-written file '%s'", path.buf); pc_item->status = PC_ITEM_FAILED; diff --git a/t/t2080-parallel-checkout-basics.sh b/t/t2080-parallel-checkout-basics.sh index 88a6ec1826224e..322d26708e0bfd 100755 --- a/t/t2080-parallel-checkout-basics.sh +++ b/t/t2080-parallel-checkout-basics.sh @@ -274,4 +274,50 @@ test_expect_success '"git checkout ." report should not include failed entries' ) ' +# Regression test: parallel checkout + fscache stale directory listing. +# +# When checkout.workers > 1, checkout_entry_ca() enqueues files for deferred +# writing instead of writing them inline. The inline write_entry() path calls +# flush_fscache() after each file, keeping the Windows fscache in sync with +# newly-created directories. The deferred path skips this flush, so +# has_dirs_only_path() sees stale ENOENT for directories that mkdir() just +# created. The recovery path in create_directories() then tries to unlink+ +# recreate the directory, which fails because it already has children. +# +# The trigger is: two files sharing a parent directory that does not yet exist +# on disk when `git checkout -- ` runs. +test_expect_success MINGW 'parallel checkout with fscache does not fail on new directories' ' + git init fscache-pc && + ( + cd fscache-pc && + git config core.fscache true && + + # Commit B1: files in a nested directory + mkdir -p sub/deep/dir && + echo one >sub/deep/dir/file1.txt && + echo two >sub/deep/dir/file2.txt && + git add sub && + git commit -m "B1: with sub/deep/dir" && + git tag B1 && + + # Commit B2: the directory is gone + git rm -rf sub && + git commit -m "B2: without sub" && + + # Now restore both files from B1 with parallel checkout. + # This is the pathspec checkout path (checkout_worktree in + # builtin/checkout.c), which defers writes via enqueue_checkout + # when workers > 1 and does not flush fscache between entries. + git -c checkout.workers=2 \ + -c checkout.thresholdForParallelism=0 \ + checkout B1 -- sub/deep/dir/file1.txt sub/deep/dir/file2.txt && + + # Verify both files are correctly restored + echo one >expect1 && + echo two >expect2 && + test_cmp expect1 sub/deep/dir/file1.txt && + test_cmp expect2 sub/deep/dir/file2.txt + ) +' + test_done From 09e66144845f4522d3ec83329b4ad8fdfd12b32c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 13 Nov 2025 11:23:29 +0100 Subject: [PATCH 124/229] ci(macos): skip the `git p4` tests Historically, the macOS jobs have always been among the longest-running ones, and recently the `git p4` tests became another liability: They started to fail much more often (maybe as of the switch away from the `macos-13` pool?), requiring re-runs of the jobs that already were responsible for long CI build times. Of the 35 test scripts that exercise `git p4`, 32 are actually run on macOS (3 are skipped for reasons like case-sensitivee filesystem), and they take an accumulated runtime of over half an hour. Furthermore, the `git p4` command is not really affected by Git for Windows' patches, at least not as far as macOS is concerned, therefore it is not only causing developer friction to have these long-running, frequently failing tests, it is also quite wasteful: There has not been a single instance so far where any `git p4` test failure in Git for Windows had demonstrated an actionable bug. While upstream Git is confident to have addressed the flakiness of the `git p4` tests via ffff0bb0dac1 (Use Perforce arm64 binary on macOS CI jobs, 2025-11-16) (which got slipped in at the 11th hour into the v2.52.0 release, fast-tracked without ever hitting `seen` even after -rc2 was released), I am not quite so confident, and besides, the runtime penalty of running those tests in Git for Windows' CI runs is still a worrisome burden. So let's just disable those tests in the CI runs, at least on macOS. Signed-off-by: Johannes Schindelin --- ci/install-dependencies.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index d57dce56630173..8606e30779b7fe 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -119,11 +119,12 @@ macos-*) # brew install gnu-time brew link --force gettext - mkdir -p "$CUSTOM_PATH" - wget -q "$P4WHENCE/bin.macosx12arm64/helix-core-server.tgz" && - tar -xf helix-core-server.tgz -C "$CUSTOM_PATH" p4 p4d && - sudo xattr -d com.apple.quarantine "$CUSTOM_PATH/p4" "$CUSTOM_PATH/p4d" 2>/dev/null || true - rm helix-core-server.tgz + # Uncomment this block if you want to run `git p4` tests: + # mkdir -p "$CUSTOM_PATH" + # wget -q "$P4WHENCE/bin.macosx12arm64/helix-core-server.tgz" && + # tar -xf helix-core-server.tgz -C "$CUSTOM_PATH" p4 p4d && + # sudo xattr -d com.apple.quarantine "$CUSTOM_PATH/p4" "$CUSTOM_PATH/p4d" 2>/dev/null || true + # rm helix-core-server.tgz case "$jobname" in osx-meson) From 152762e89d978d6f5b148ea99d8fda4373ad3bff Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 23 Feb 2018 02:50:03 +0100 Subject: [PATCH 125/229] mingw (git_terminal_prompt): do fall back to CONIN$/CONOUT$ method To support Git Bash running in a MinTTY, we use a dirty trick to access the MSYS2 pseudo terminal: we execute a Bash snippet that accesses /dev/tty. The idea was to fall back to writing to/reading from CONOUT$/CONIN$ if that Bash call failed because Bash was not found. However, we should fall back even in other error conditions, because we have not successfully read the user input. Let's make it so. Signed-off-by: Johannes Schindelin --- compat/terminal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compat/terminal.c b/compat/terminal.c index a89c5cd9ccf604..882b027e41e52b 100644 --- a/compat/terminal.c +++ b/compat/terminal.c @@ -483,7 +483,7 @@ char *git_terminal_prompt(const char *prompt, int echo) /* try shell_prompt first, fall back to CONIN/OUT if bash is missing */ char *result = shell_prompt(prompt, echo); - if (result || errno != ENOENT) + if (result) return result; #endif From 8644fa2f38814a92c316c701a1de20435a62b44a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 23 Apr 2018 23:20:00 +0200 Subject: [PATCH 126/229] mingw: Windows Docker volumes are *not* symbolic links ... even if they may look like them. As looking up the target of the "symbolic link" (just to see whether it starts with `/ContainerMappedDirectories/`) is pretty expensive, we do it when we can be *really* sure that there is a possibility that this might be the case. Signed-off-by: Johannes Schindelin Signed-off-by: JiSeop Moon --- compat/mingw.c | 25 +++++++++++++++++++------ compat/win32.h | 2 +- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 9b6ad28b05243d..9aa5c96ed1aa73 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1269,7 +1269,7 @@ int mingw_lstat(const char *file_name, struct stat *buf) buf->st_uid = 0; buf->st_nlink = 1; buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, - reparse_tag); + reparse_tag, file_name); buf->st_size = S_ISLNK(buf->st_mode) ? link_len : fdata.nFileSizeLow | (((off_t) fdata.nFileSizeHigh) << 32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ @@ -1318,7 +1318,7 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) buf->st_gid = 0; buf->st_uid = 0; buf->st_nlink = 1; - buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, 0); + buf->st_mode = file_attr_to_st_mode(fdata.dwFileAttributes, 0, NULL); buf->st_size = fdata.nFileSizeLow | (((off_t)fdata.nFileSizeHigh)<<32); buf->st_dev = buf->st_rdev = 0; /* not used by Git */ @@ -3893,12 +3893,25 @@ int is_inside_windows_container(void) return inside_container; } -int file_attr_to_st_mode (DWORD attr, DWORD tag) +int file_attr_to_st_mode (DWORD attr, DWORD tag, const char *path) { int fMode = S_IREAD; - if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && tag == IO_REPARSE_TAG_SYMLINK) - fMode |= S_IFLNK; - else if (attr & FILE_ATTRIBUTE_DIRECTORY) + if ((attr & FILE_ATTRIBUTE_REPARSE_POINT) && + tag == IO_REPARSE_TAG_SYMLINK) { + int flag = S_IFLNK; + char buf[MAX_LONG_PATH]; + + /* + * Windows containers' mapped volumes are marked as reparse + * points and look like symbolic links, but they are not. + */ + if (path && is_inside_windows_container() && + readlink(path, buf, sizeof(buf)) > 27 && + starts_with(buf, "/ContainerMappedDirectories/")) + flag = S_IFDIR; + + fMode |= flag; + } else if (attr & FILE_ATTRIBUTE_DIRECTORY) fMode |= S_IFDIR; else fMode |= S_IFREG; diff --git a/compat/win32.h b/compat/win32.h index 52169ae19f4371..299f01bdf0f5a4 100644 --- a/compat/win32.h +++ b/compat/win32.h @@ -6,7 +6,7 @@ #include #endif -extern int file_attr_to_st_mode (DWORD attr, DWORD tag); +extern int file_attr_to_st_mode (DWORD attr, DWORD tag, const char *path); static inline int get_file_attr(const char *fname, WIN32_FILE_ATTRIBUTE_DATA *fdata) { From c9daa88603b345600825acc086d3aca9660d417c Mon Sep 17 00:00:00 2001 From: David Lomas Date: Fri, 28 Jul 2023 15:20:43 +0100 Subject: [PATCH 127/229] mingw: work around rename() failing on a read-only file At least on _some_ APFS network shares, Git fails to rename the object files because they are marked as read-only, because that has the effect of setting the uchg flag on APFS, which then means the file can't be renamed or deleted. To work around that, when a rename failed, and the read-only flag is set, try to turn it off and on again. This fixes https://github.com/git-for-windows/git/issues/4482 Signed-off-by: David Lomas Signed-off-by: Johannes Schindelin --- compat/mingw.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 9aa5c96ed1aa73..680302d47372ce 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2527,7 +2527,7 @@ int mingw_accept(int sockfd1, struct sockaddr *sa, socklen_t *sz) int mingw_rename(const char *pold, const char *pnew) { static int supports_file_rename_info_ex = 1; - DWORD attrs = INVALID_FILE_ATTRIBUTES, gle; + DWORD attrs = INVALID_FILE_ATTRIBUTES, gle, attrsold; int tries = 0; wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; int wpnew_len; @@ -2619,11 +2619,24 @@ int mingw_rename(const char *pold, const char *pnew) gle = GetLastError(); } - if (gle == ERROR_ACCESS_DENIED && is_inside_windows_container()) { - /* Fall back to copy to destination & remove source */ - if (CopyFileW(wpold, wpnew, FALSE) && !mingw_unlink(pold, 1)) - return 0; - gle = GetLastError(); + if (gle == ERROR_ACCESS_DENIED) { + if (is_inside_windows_container()) { + /* Fall back to copy to destination & remove source */ + if (CopyFileW(wpold, wpnew, FALSE) && !mingw_unlink(pold, 1)) + return 0; + gle = GetLastError(); + } else if ((attrsold = GetFileAttributesW(wpold)) & FILE_ATTRIBUTE_READONLY) { + /* if file is read-only, change and retry */ + SetFileAttributesW(wpold, attrsold & ~FILE_ATTRIBUTE_READONLY); + if (MoveFileExW(wpold, wpnew, + MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED)) { + SetFileAttributesW(wpnew, attrsold); + return 0; + } + gle = GetLastError(); + /* revert attribute change on failure */ + SetFileAttributesW(wpold, attrsold); + } } /* revert file attributes on failure */ From 7396bf34790d76bc830b509d91452f84e559436c Mon Sep 17 00:00:00 2001 From: Bert Belder Date: Fri, 26 Oct 2018 23:42:09 +0200 Subject: [PATCH 128/229] Win32: symlink: add test for `symlink` attribute To verify that the symlink is resolved correctly, we use the fact that `git.exe` is a native Win32 program, and that `git.exe config -f ` therefore uses the native symlink resolution. Signed-off-by: Bert Belder Signed-off-by: Johannes Schindelin --- t/meson.build | 1 + t/t2040-checkout-symlink-attr.sh | 46 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100755 t/t2040-checkout-symlink-attr.sh diff --git a/t/meson.build b/t/meson.build index ed252313c96c74..3eff50fbf79dd5 100644 --- a/t/meson.build +++ b/t/meson.build @@ -287,6 +287,7 @@ integration_tests = [ 't2026-checkout-pathspec-file.sh', 't2027-checkout-track.sh', 't2030-unresolve-info.sh', + 't2040-checkout-symlink-attr.sh', 't2050-git-dir-relative.sh', 't2060-switch.sh', 't2070-restore.sh', diff --git a/t/t2040-checkout-symlink-attr.sh b/t/t2040-checkout-symlink-attr.sh new file mode 100755 index 00000000000000..e00c31d096ce88 --- /dev/null +++ b/t/t2040-checkout-symlink-attr.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +test_description='checkout symlinks with `symlink` attribute on Windows + +Ensures that Git for Windows creates symlinks of the right type, +as specified by the `symlink` attribute in `.gitattributes`.' + +# Tell MSYS to create native symlinks. Without this flag test-lib's +# prerequisite detection for SYMLINKS doesn't detect the right thing. +MSYS=winsymlinks:nativestrict && export MSYS + +. ./test-lib.sh + +if ! test_have_prereq MINGW,SYMLINKS +then + skip_all='skipping $0: MinGW-only test, which requires symlink support.' + test_done +fi + +# Adds a symlink to the index without clobbering the work tree. +cache_symlink () { + sha=$(printf '%s' "$1" | git hash-object --stdin -w) && + git update-index --add --cacheinfo 120000,$sha,"$2" +} + +test_expect_success 'checkout symlinks with attr' ' + cache_symlink file1 file-link && + cache_symlink dir dir-link && + + printf "file-link symlink=file\ndir-link symlink=dir\n" >.gitattributes && + git add .gitattributes && + + git checkout . && + + mkdir dir && + echo "[a]b=c" >file1 && + echo "[x]y=z" >dir/file2 && + + # MSYS2 is very forgiving, it will resolve symlinks even if the + # symlink type is incorrect. To make this test meaningful, try + # them with a native, non-MSYS executable, such as `git config`. + test "$(git config -f file-link a.b)" = "c" && + test "$(git config -f dir-link/file2 x.y)" = "z" +' + +test_done From 0c78bd28a3613aa6c350bb2e3b65b693e33fe99b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 11 Dec 2018 12:55:26 +0100 Subject: [PATCH 129/229] clean: remove mount points when possible Windows' equivalent to "bind mounts", NTFS junction points, can be unlinked without affecting the mount target. This is clearly what users expect to happen when they call `git clean -dfx` in a worktree that contains NTFS junction points: the junction should be removed, and the target directory of said junction should be left alone (unless it is inside the worktree). Signed-off-by: Johannes Schindelin --- builtin/clean.c | 13 +++++++++++++ compat/mingw.h | 1 + t/t7300-clean.sh | 1 + 3 files changed, 15 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index e4f2d56d3210ba..6ed555000f9a41 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -41,8 +41,10 @@ static const char *msg_remove = N_("Removing %s\n"); static const char *msg_would_remove = N_("Would remove %s\n"); static const char *msg_skip_git_dir = N_("Skipping repository %s\n"); static const char *msg_would_skip_git_dir = N_("Would skip repository %s\n"); +#ifndef CAN_UNLINK_MOUNT_POINTS static const char *msg_skip_mount_point = N_("Skipping mount point %s\n"); static const char *msg_would_skip_mount_point = N_("Would skip mount point %s\n"); +#endif static const char *msg_warn_remove_failed = N_("failed to remove %s"); static const char *msg_warn_lstat_failed = N_("could not lstat %s\n"); static const char *msg_skip_cwd = N_("Refusing to remove current working directory\n"); @@ -188,6 +190,7 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, } if (is_mount_point(path)) { +#ifndef CAN_UNLINK_MOUNT_POINTS if (!quiet) { quote_path(path->buf, prefix, "ed, 0); printf(dry_run ? @@ -195,6 +198,16 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, _(msg_skip_mount_point), quoted.buf); } *dir_gone = 0; +#else + if (!dry_run && unlink(path->buf)) { + int saved_errno = errno; + quote_path(path->buf, prefix, "ed, 0); + errno = saved_errno; + warning_errno(_(msg_warn_remove_failed), quoted.buf); + *dir_gone = 0; + ret = -1; + } +#endif goto out; } diff --git a/compat/mingw.h b/compat/mingw.h index 92237e14c4a87c..4466cf092af263 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -39,6 +39,7 @@ static inline void convert_slashes(char *path) struct strbuf; int mingw_is_mount_point(struct strbuf *path); #define is_mount_point mingw_is_mount_point +#define CAN_UNLINK_MOUNT_POINTS 1 #define PATH_SEP ';' char *mingw_query_user_email(void); #define query_user_email mingw_query_user_email diff --git a/t/t7300-clean.sh b/t/t7300-clean.sh index af9c05255ca040..f93c95adb8607a 100755 --- a/t/t7300-clean.sh +++ b/t/t7300-clean.sh @@ -806,6 +806,7 @@ test_expect_success MINGW 'clean does not traverse mount points' ' git init with-mountpoint && cmd //c "mklink /j with-mountpoint\\mountpoint target" && git -C with-mountpoint clean -dfx && + test_path_is_missing with-mountpoint/mountpoint && test_path_is_file target/dont-clean-me ' From 7b0342105ee9baa5e4edbf040f95cb9c555392b7 Mon Sep 17 00:00:00 2001 From: xungeng li Date: Wed, 7 Jun 2023 20:26:33 +0800 Subject: [PATCH 130/229] mingw: optionally enable wsl compability file mode bits The Windows Subsystem for Linux (WSL) version 2 allows to use `chmod` on NTFS volumes provided that they are mounted with metadata enabled (see https://devblogs.microsoft.com/commandline/chmod-chown-wsl-improvements/ for details), for example: $ chmod 0755 /mnt/d/test/a.sh In order to facilitate better collaboration between the Windows version of Git and the WSL version of Git, we can make the Windows version of Git also support reading and writing NTFS file modes in a manner compatible with WSL. Since this slightly slows down operations where lots of files are created (such as an initial checkout), this feature is only enabled when `core.WSLCompat` is set to true. Note that you also have to set `core.fileMode=true` in repositories that have been initialized without enabling WSL compatibility. There are several ways to enable metadata loading for NTFS volumes in WSL, one of which is to modify `/etc/wsl.conf` by adding: ``` [automount] enabled = true options = "metadata,umask=027,fmask=117" ``` And reboot WSL. It can also be enabled temporarily by this incantation: $ sudo umount /mnt/c && sudo mount -t drvfs C: /mnt/c -o metadata,uid=1000,gid=1000,umask=22,fmask=111 It's important to note that this modification is compatible with, but does not depend on WSL. The helper functions in this commit can operate independently and functions normally on devices where WSL is not installed or properly configured. Signed-off-by: xungeng li Signed-off-by: Johannes Schindelin --- Documentation/config/core.adoc | 6 ++ compat/mingw.c | 13 +++ compat/win32/wsl.c | 142 ++++++++++++++++++++++++++++ compat/win32/wsl.h | 12 +++ config.mak.uname | 4 +- contrib/buildsystems/CMakeLists.txt | 1 + meson.build | 1 + 7 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 compat/win32/wsl.c create mode 100644 compat/win32/wsl.h diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index f586fb061cab6d..aaafe7a7889f66 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -819,3 +819,9 @@ core.maxTreeDepth:: to allow Git to abort cleanly, and should not generally need to be adjusted. When Git is compiled with MSVC, the default is 512. Otherwise, the default is 2048. + +core.WSLCompat:: + Tells Git whether to enable wsl compatibility mode. + The default value is false. When set to true, Git will set the mode + bits of the file in the way of wsl, so that the executable flag of + files can be set or read correctly. diff --git a/compat/mingw.c b/compat/mingw.c index 93eb3509544092..ab82fadca5aefc 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -16,6 +16,7 @@ #include "win32.h" #include "win32/exit-process.h" #include "win32/lazyload.h" +#include "win32/wsl.h" #include "wrapper.h" #include #include @@ -895,6 +896,11 @@ int mingw_open (const char *filename, int oflags, ...) if (fd < 0 && create && GetLastError() == ERROR_ACCESS_DENIED && INIT_PROC_ADDR(RtlGetLastNtStatus) && RtlGetLastNtStatus() == STATUS_DELETE_PENDING) errno = EEXIST; + else if ((oflags & O_CREAT) && fd >= 0 && are_wsl_compatible_mode_bits_enabled()) { + _mode_t wsl_mode = S_IFREG | (mode&0777); + set_wsl_mode_bits_by_handle((HANDLE)_get_osfhandle(fd), wsl_mode); + } + if (fd < 0 && (oflags & O_ACCMODE) != O_RDONLY && errno == EACCES) { DWORD attrs = GetFileAttributesW(wfilename); if (attrs != INVALID_FILE_ATTRIBUTES && (attrs & FILE_ATTRIBUTE_DIRECTORY)) @@ -1276,6 +1282,11 @@ int mingw_lstat(const char *file_name, struct stat *buf) filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); + if (S_ISREG(buf->st_mode) && + are_wsl_compatible_mode_bits_enabled()) { + copy_wsl_mode_bits_from_disk(wfilename, -1, + &buf->st_mode); + } return 0; } @@ -1325,6 +1336,8 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) filetime_to_timespec(&(fdata.ftLastAccessTime), &(buf->st_atim)); filetime_to_timespec(&(fdata.ftLastWriteTime), &(buf->st_mtim)); filetime_to_timespec(&(fdata.ftCreationTime), &(buf->st_ctim)); + if (are_wsl_compatible_mode_bits_enabled()) + get_wsl_mode_bits_by_handle(hnd, &buf->st_mode); return 0; } diff --git a/compat/win32/wsl.c b/compat/win32/wsl.c new file mode 100644 index 00000000000000..ab599770138b4e --- /dev/null +++ b/compat/win32/wsl.c @@ -0,0 +1,142 @@ +#define USE_THE_REPOSITORY_VARIABLE +#include "../../git-compat-util.h" +#include "../win32.h" +#include "../../repository.h" +#include "config.h" +#include "ntifs.h" +#include "wsl.h" + +int are_wsl_compatible_mode_bits_enabled(void) +{ + /* default to `false` during initialization */ + static const int fallback = 0; + static int enabled = -1; + + if (enabled < 0) { + /* avoid infinite recursion */ + if (!the_repository) + return fallback; + + if (the_repository->config && + the_repository->config->hash_initialized && + repo_config_get_bool(the_repository, "core.wslcompat", &enabled) < 0) + enabled = 0; + } + + return enabled < 0 ? fallback : enabled; +} + +int copy_wsl_mode_bits_from_disk(const wchar_t *wpath, ssize_t wpathlen, + _mode_t *mode) +{ + int ret = -1; + HANDLE h; + if (wpathlen >= 0) { + /* + * It's caller's duty to make sure wpathlen is reasonable so + * it does not overflow. + */ + wchar_t *fn2 = (wchar_t*)alloca((wpathlen + 1) * sizeof(wchar_t)); + memcpy(fn2, wpath, wpathlen * sizeof(wchar_t)); + fn2[wpathlen] = 0; + wpath = fn2; + } + h = CreateFileW(wpath, FILE_READ_EA | SYNCHRONIZE, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS | + FILE_FLAG_OPEN_REPARSE_POINT, + NULL); + if (h != INVALID_HANDLE_VALUE) { + ret = get_wsl_mode_bits_by_handle(h, mode); + CloseHandle(h); + } + return ret; +} + +#ifndef LX_FILE_METADATA_HAS_UID +#define LX_FILE_METADATA_HAS_UID 0x1 +#define LX_FILE_METADATA_HAS_GID 0x2 +#define LX_FILE_METADATA_HAS_MODE 0x4 +#define LX_FILE_METADATA_HAS_DEVICE_ID 0x8 +#define LX_FILE_CASE_SENSITIVE_DIR 0x10 +typedef struct _FILE_STAT_LX_INFORMATION { + LARGE_INTEGER FileId; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER AllocationSize; + LARGE_INTEGER EndOfFile; + uint32_t FileAttributes; + uint32_t ReparseTag; + uint32_t NumberOfLinks; + ACCESS_MASK EffectiveAccess; + uint32_t LxFlags; + uint32_t LxUid; + uint32_t LxGid; + uint32_t LxMode; + uint32_t LxDeviceIdMajor; + uint32_t LxDeviceIdMinor; +} FILE_STAT_LX_INFORMATION, *PFILE_STAT_LX_INFORMATION; +#endif + +/* + * This struct is extended from the original FILE_FULL_EA_INFORMATION of + * Microsoft Windows. + */ +struct wsl_full_ea_info_t { + uint32_t NextEntryOffset; + uint8_t Flags; + uint8_t EaNameLength; + uint16_t EaValueLength; + char EaName[7]; + char EaValue[4]; + char Padding[1]; +}; + +enum { + FileStatLxInformation = 70, +}; +__declspec(dllimport) NTSTATUS WINAPI + NtQueryInformationFile(HANDLE FileHandle, + PIO_STATUS_BLOCK IoStatusBlock, + PVOID FileInformation, ULONG Length, + uint32_t FileInformationClass); +__declspec(dllimport) NTSTATUS WINAPI + NtSetInformationFile(HANDLE FileHandle, PIO_STATUS_BLOCK IoStatusBlock, + PVOID FileInformation, ULONG Length, + uint32_t FileInformationClass); +__declspec(dllimport) NTSTATUS WINAPI + NtSetEaFile(HANDLE FileHandle, PIO_STATUS_BLOCK IoStatusBlock, + PVOID EaBuffer, ULONG EaBufferSize); + +int set_wsl_mode_bits_by_handle(HANDLE h, _mode_t mode) +{ + uint32_t value = mode; + struct wsl_full_ea_info_t ea_info; + IO_STATUS_BLOCK iob; + /* mode should be valid to make WSL happy */ + assert(S_ISREG(mode) || S_ISDIR(mode)); + ea_info.NextEntryOffset = 0; + ea_info.Flags = 0; + ea_info.EaNameLength = 6; + ea_info.EaValueLength = sizeof(value); /* 4 */ + strlcpy(ea_info.EaName, "$LXMOD", sizeof(ea_info.EaName)); + memcpy(ea_info.EaValue, &value, sizeof(value)); + ea_info.Padding[0] = 0; + return NtSetEaFile(h, &iob, &ea_info, sizeof(ea_info)); +} + +int get_wsl_mode_bits_by_handle(HANDLE h, _mode_t *mode) +{ + FILE_STAT_LX_INFORMATION fxi; + IO_STATUS_BLOCK iob; + if (NtQueryInformationFile(h, &iob, &fxi, sizeof(fxi), + FileStatLxInformation) == 0) { + if (fxi.LxFlags & LX_FILE_METADATA_HAS_MODE) + *mode = (_mode_t)fxi.LxMode; + return 0; + } + return -1; +} diff --git a/compat/win32/wsl.h b/compat/win32/wsl.h new file mode 100644 index 00000000000000..1f5ad7e67a4fc2 --- /dev/null +++ b/compat/win32/wsl.h @@ -0,0 +1,12 @@ +#ifndef COMPAT_WIN32_WSL_H +#define COMPAT_WIN32_WSL_H + +int are_wsl_compatible_mode_bits_enabled(void); + +int copy_wsl_mode_bits_from_disk(const wchar_t *wpath, ssize_t wpathlen, + _mode_t *mode); + +int get_wsl_mode_bits_by_handle(HANDLE h, _mode_t *mode); +int set_wsl_mode_bits_by_handle(HANDLE h, _mode_t mode); + +#endif diff --git a/config.mak.uname b/config.mak.uname index 4d5c8d321ed59c..50651d7612478e 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -538,7 +538,7 @@ endif compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/trace2_win32_process_info.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/wsl.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY \ -DENSURE_MSYSTEM_IS_SET="\"$(MSYSTEM)\"" -DMINGW_PREFIX="\"$(patsubst /%,%,$(MINGW_PREFIX))\"" \ -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" @@ -743,7 +743,7 @@ ifeq ($(uname_S),MINGW) compat/win32/flush.o \ compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o + compat/win32/dirent.o compat/win32/wsl.o BASIC_CFLAGS += -DWIN32 EXTLIBS += -lws2_32 GITLIBS += git.res diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index d494d91e67c9f3..92fb5149c38247 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -274,6 +274,7 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") compat/win32/syslog.c compat/win32/trace2_win32_process_info.c compat/win32/dirent.c + compat/win32/wsl.c compat/strdup.c) set(NO_UNIX_SOCKETS 1) diff --git a/meson.build b/meson.build index a2d373b37352a3..247ec0e05d0d41 100644 --- a/meson.build +++ b/meson.build @@ -1317,6 +1317,7 @@ elif host_machine.system() == 'windows' 'compat/win32/path-utils.c', 'compat/win32/pthread.c', 'compat/win32/syslog.c', + 'compat/win32/wsl.c', 'compat/win32mmap.c', ] From aec4f855ceac786d17168587bd2a01a0e7485cff Mon Sep 17 00:00:00 2001 From: Sven Strickroth Date: Thu, 28 May 2026 19:04:50 +0200 Subject: [PATCH 131/229] Refuse to follow invalid paths in `.git` files This corresponds to the fixes in TortoiseGit v2.19 that were announced in https://groups.google.com/g/tortoisegit-announce/c/31zdmOBi4vY. The threat surface in Git for Windows is a lot smaller than in TortoiseGit because Git is not integrated directly into the Windows Explorer and hence does not run automatically on any extracted archive. A Git user would have to run Git operations on an extracted archive of dubious origin, which is never a good idea. In all other circumstances, the `.git` files Git would see would always be written by Git for Windows itself, which will never write `.git` files containing invalid paths (or paths pointing to other servers). For this reason, it was deemed okay to integrate this change as a regular patch. Signed-off-by: Sven Strickroth Signed-off-by: Johannes Schindelin --- compat/mingw.c | 33 +++++++++++++++++++++++++++++++++ compat/mingw.h | 1 + setup.c | 39 +++++++++++++++++++++++++++++++++++++++ t/t5580-unc-paths.sh | 2 ++ 4 files changed, 75 insertions(+) diff --git a/compat/mingw.c b/compat/mingw.c index 0c8179f3458bb8..37c3e17ab0a0b1 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3374,6 +3374,30 @@ static int acls_supported(const char *path) return 0; } +int is_valid_windows_path_element(wchar_t ch) +{ + // cf. https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file + // Windows disallows ASCII control characters 0x00–0x1F + if (ch < 0x1F) + return false; + + // Windows reserved path characters + switch (ch) { + case L'<': + case L'>': + case L':': + case L'"': + case L'/': + case L'\\': + case L'|': + case L'?': + case L'*': + return 0; + default: + return 1; + } +} + int is_path_owned_by_current_sid(const char *path, struct strbuf *report) { WCHAR wpath[MAX_PATH]; @@ -3402,6 +3426,15 @@ int is_path_owned_by_current_sid(const char *path, struct strbuf *report) if (!wcsicmp(wpath, home)) return 1; + /* Do not leak NTLM hashes, UNC paths etc. are generally problematic */ + if (wpath[0] == L'\\' && !is_valid_windows_path_element(wpath[1])) { + if (report) + strbuf_addf(report, + "'%s' may refer to a non-local directory", + path); + return 0; + } + /* Get the owner SID */ err = GetNamedSecurityInfoW(wpath, SE_FILE_OBJECT, OWNER_SECURITY_INFORMATION | diff --git a/compat/mingw.h b/compat/mingw.h index df02aeb632d8c3..aac1148be3e619 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -46,6 +46,7 @@ char *mingw_query_user_email(void); */ int is_path_owned_by_current_sid(const char *path, struct strbuf *report); #define is_path_owned_by_current_user is_path_owned_by_current_sid +int is_valid_windows_path_element(wchar_t ch); /** * Verifies that the given path is a valid one on Windows. diff --git a/setup.c b/setup.c index 61bcdafe82a20b..7f8c68adccfe30 100644 --- a/setup.c +++ b/setup.c @@ -933,6 +933,35 @@ void read_gitfile_error_die(int error_code, const char *path) } } +#if (defined _WIN32 || defined __WIN32__) +static int ensure_valid_ownership(const char *gitfile, + const char *worktree, const char *gitdir, + struct strbuf *report); + +static int is_invalid_dotgit_path(const char *gitfile, const char *potential_gitdir) +{ + WCHAR wpath[MAX_PATH]; + struct strbuf dir = STRBUF_INIT; + int ret; + + if (xutftowcs_path(wpath, potential_gitdir) < 0) + return 1; + + /* Do not leak NTLM hashes, UNC paths etc. are generally problematic */ + if (wpath[0] != L'\\' || is_valid_windows_path_element(wpath[1])) + return 0; + + strbuf_addstr(&dir, gitfile); + strbuf_strip_file_from_path(&dir); + + ret = ensure_valid_ownership(gitfile, dir.buf, potential_gitdir, NULL) ? 0 : 1; + + strbuf_release(&dir); + + return ret; +} +#endif + /* * Try to read the location of the git directory from the .git file, * return path to git directory if found. The return value comes from @@ -961,11 +990,21 @@ const char *read_gitfile_gently(const char *path, int *return_error_code) strbuf_addstr(&contents, dir); free(dir); } +#if (defined _WIN32 || defined __WIN32__) + if (is_dir_sep(contents.buf[0]) && + is_invalid_dotgit_path(path, contents.buf)) { + strbuf_reset(&realpath); + strbuf_add(&realpath, contents.buf, contents.len); + path = realpath.buf; + goto cleanup_return; + } +#endif if (!is_git_directory(contents.buf)) { error_code = READ_GITFILE_ERR_NOT_A_REPO; goto cleanup_return; } + strbuf_reset(&realpath); strbuf_realpath(&realpath, contents.buf, 1); cleanup_return: diff --git a/t/t5580-unc-paths.sh b/t/t5580-unc-paths.sh index 65ef1a3628ee94..a2c7f9853ac833 100755 --- a/t/t5580-unc-paths.sh +++ b/t/t5580-unc-paths.sh @@ -44,6 +44,8 @@ test_expect_success clone ' ' test_expect_success 'clone without file://' ' + test_must_fail git clone "$UNCPATH" clone-without-file && + git config set --global --append safe.directory "$UNCPATH/.git" && git clone "$UNCPATH" clone-without-file ' From 808a65688a882e13be3dfe649af5d10e41ba10e7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 19:46:47 +0200 Subject: [PATCH 132/229] tree-walk: drop link_len cast in get_tree_entry_follow_symlinks() Smallest piece of the tree topic. link_len is only used as strbuf_splice()'s size_t length and as an array index; widening it outright removes the cast_size_t_to_ulong() shim and the bridge local that fed it. odb_read_object() now writes straight into link_len. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- tree-walk.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tree-walk.c b/tree-walk.c index a67f06b9ebd790..a7bbe3163ac9f3 100644 --- a/tree-walk.c +++ b/tree-walk.c @@ -777,8 +777,7 @@ enum get_oid_result get_tree_entry_follow_symlinks(struct repository *r, goto done; } else if (S_ISLNK(*mode)) { /* Follow a symlink */ - unsigned long link_len; - size_t link_len_st = 0; + size_t link_len; size_t len; char *contents, *contents_start; struct dir_state *parent; @@ -798,8 +797,7 @@ enum get_oid_result get_tree_entry_follow_symlinks(struct repository *r, contents = odb_read_object(r->objects, ¤t_tree_oid, &type, - &link_len_st); - link_len = cast_size_t_to_ulong(link_len_st); + &link_len); if (!contents) goto done; From 001f4db821de4a12af8f165b47b4541a85f69cf3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 20:11:56 +0200 Subject: [PATCH 133/229] tree-walk: widen init_tree_desc() and init_tree_desc_gently() to size_t Prep for dropping the cast_size_t_to_ulong() shim in add_preferred_base() (pack-objects.c), and aligns the public API with the size_t shape the rest of the tree topic is moving toward. struct tree_desc.size stays unsigned int -- the on-disk tree format hard-caps each tree at 4 GiB, so the field is intentionally narrow and the assignment in init_tree_desc_internal() already truncated unsigned long inputs the same way it now truncates size_t inputs. The widening is purely about the call-side type-correctness; the internal cap is unchanged. All 30+ callers pass values that promote cleanly (unsigned long, size_t, or smaller integer types). Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- tree-walk.c | 6 +++--- tree-walk.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tree-walk.c b/tree-walk.c index a7bbe3163ac9f3..e2cea5d8834afc 100644 --- a/tree-walk.c +++ b/tree-walk.c @@ -49,7 +49,7 @@ static int decode_tree_entry(struct tree_desc *desc, const char *buf, unsigned l static int init_tree_desc_internal(struct tree_desc *desc, const struct object_id *oid, - const void *buffer, unsigned long size, + const void *buffer, size_t size, struct strbuf *err, enum tree_desc_flags flags) { @@ -63,7 +63,7 @@ static int init_tree_desc_internal(struct tree_desc *desc, } void init_tree_desc(struct tree_desc *desc, const struct object_id *tree_oid, - const void *buffer, unsigned long size) + const void *buffer, size_t size) { struct strbuf err = STRBUF_INIT; if (init_tree_desc_internal(desc, tree_oid, buffer, size, &err, 0)) @@ -72,7 +72,7 @@ void init_tree_desc(struct tree_desc *desc, const struct object_id *tree_oid, } int init_tree_desc_gently(struct tree_desc *desc, const struct object_id *oid, - const void *buffer, unsigned long size, + const void *buffer, size_t size, enum tree_desc_flags flags) { struct strbuf err = STRBUF_INIT; diff --git a/tree-walk.h b/tree-walk.h index 9646c47ac5bce5..af6e82fd3f01ba 100644 --- a/tree-walk.h +++ b/tree-walk.h @@ -85,10 +85,10 @@ int update_tree_entry_gently(struct tree_desc *); * members of `struct tree`. */ void init_tree_desc(struct tree_desc *desc, const struct object_id *tree_oid, - const void *buf, unsigned long size); + const void *buf, size_t size); int init_tree_desc_gently(struct tree_desc *desc, const struct object_id *oid, - const void *buf, unsigned long size, + const void *buf, size_t size, enum tree_desc_flags flags); /* From b751a3aaf2a571fa435d53ad3a2fa21bec6c3ca5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 20:23:53 +0200 Subject: [PATCH 134/229] pack-objects: drop the two tree-walk casts in the preferred-base path With init_tree_desc() widened in the prior commit, the size_t-returning odb_read_object_peeled() call in add_preferred_base() and odb_read_object() call in pbase_tree_get() can both flow straight through to init_tree_desc() and into the pbase_tree_cache. Widen pbase_tree_cache.tree_size and the two local size variables to size_t, drop the size_st bridges, and drop the two cast_size_t_to_ulong() shims. This was the last pair of cast_size_t_to_ulong() call sites in builtin/pack-objects.c, completing the >4 GiB-objects work in that file that this branch and its predecessors have been pursuing. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/pack-objects.c | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index b66fffe092e2f0..ab15529a9dabda 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1915,7 +1915,7 @@ struct pbase_tree_cache { int ref; int temporary; void *tree_data; - unsigned long tree_size; + size_t tree_size; }; static struct pbase_tree_cache *(pbase_tree_cache[256]); @@ -1942,8 +1942,7 @@ static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid) { struct pbase_tree_cache *ent, *nent; void *data; - unsigned long size; - size_t size_st = 0; + size_t size; enum object_type type; int neigh; int my_ix = pbase_tree_cache_ix(oid); @@ -1971,8 +1970,7 @@ static struct pbase_tree_cache *pbase_tree_get(const struct object_id *oid) /* Did not find one. Either we got a bogus request or * we need to read and perhaps cache. */ - data = odb_read_object(the_repository->objects, oid, &type, &size_st); - size = cast_size_t_to_ulong(size_st); + data = odb_read_object(the_repository->objects, oid, &type, &size); if (!data) return NULL; if (type != OBJ_TREE) { @@ -2126,16 +2124,14 @@ static void add_preferred_base(struct object_id *oid) { struct pbase_tree *it; void *data; - unsigned long size; - size_t size_st = 0; + size_t size; struct object_id tree_oid; if (window <= num_preferred_base++) return; data = odb_read_object_peeled(the_repository->objects, oid, - OBJ_TREE, &size_st, &tree_oid); - size = cast_size_t_to_ulong(size_st); + OBJ_TREE, &size, &tree_oid); if (!data) return; From e01be6622c7a1eb930e8226ba288927869a7ca96 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 19:38:13 +0200 Subject: [PATCH 135/229] diffcore: widen struct diff_filespec.size to size_t Continue the size_t evacuation. The struct field already receives its writes from a size_t-shaped source (xsize_t(st.st_size), strbuf.len, fill_textconv()'s return, odb_read_object_info_extended() via oi.sizep), so on Windows it was already truncating anything past 4 GiB silently on the strbuf and textconv paths and loudly through cast_size_t_to_ulong() on the odb path. Switch the field to size_t. In diff_populate_filespec(), point oi.sizep at the field directly and drop both cast_size_t_to_ulong() shims and the size_st bridge they fed. Downstream consumers that still read .size into unsigned long locals will now silently narrow on Windows where the field exceeds 4 GiB. Each of those is its own follow-up; the writer side is the prerequisite for ever putting a >4 GiB value in the field in the first place. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- diff.c | 5 +---- diffcore.h | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/diff.c b/diff.c index 5e7834851d0852..0b825070fa469c 100644 --- a/diff.c +++ b/diff.c @@ -4772,9 +4772,8 @@ int diff_populate_filespec(struct repository *r, } } else { - size_t size_st = 0; struct object_info info = { - .sizep = &size_st + .sizep = &s->size }; if (!(size_only || check_binary)) @@ -4796,7 +4795,6 @@ int diff_populate_filespec(struct repository *r, die("unable to read %s", oid_to_hex(&s->oid)); object_read: - s->size = cast_size_t_to_ulong(size_st); if (size_only || check_binary) { if (size_only) return 0; @@ -4811,7 +4809,6 @@ int diff_populate_filespec(struct repository *r, if (odb_read_object_info_extended(r->objects, &s->oid, &info, OBJECT_INFO_LOOKUP_REPLACE)) die("unable to read %s", oid_to_hex(&s->oid)); - s->size = cast_size_t_to_ulong(size_st); } s->should_free = 1; } diff --git a/diffcore.h b/diffcore.h index d75038d1b3a1b2..85fc94e2a5d603 100644 --- a/diffcore.h +++ b/diffcore.h @@ -54,7 +54,7 @@ struct diff_filespec { char *path; void *data; void *cnt_data; - unsigned long size; + size_t size; int count; /* Reference count */ int rename_used; /* Count of rename users */ unsigned short mode; /* file mode */ From 14bdd3ddd5fec768fa14f121ff5e17b5b41e6043 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 21:27:35 +0200 Subject: [PATCH 136/229] tree: widen struct tree.size and parse_tree_buffer() to size_t Final piece of the tree topic. struct tree.size already receives its values from size_t-shaped sources (odb_read_object() in repo_parse_tree_gently() and in reflog.c::tree_is_complete()), so on Windows it was already silently truncating anything past 4 GiB. Switch the field and parse_tree_buffer()'s size parameter to size_t. All readers feed tree->size into init_tree_desc(), which was widened earlier in this topic; the existing parse_object_buffer() caller in object.c keeps its unsigned long parameter, which promotes cleanly. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- tree.c | 2 +- tree.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tree.c b/tree.c index 53f7395e9fea09..d37b9bc7b197dd 100644 --- a/tree.c +++ b/tree.c @@ -172,7 +172,7 @@ struct tree *lookup_tree(struct repository *r, const struct object_id *oid) return object_as_type(obj, OBJ_TREE, 0); } -int parse_tree_buffer(struct tree *item, void *buffer, unsigned long size) +int parse_tree_buffer(struct tree *item, void *buffer, size_t size) { if (item->object.parsed) return 0; diff --git a/tree.h b/tree.h index 677382eed86c59..50f0b15af45326 100644 --- a/tree.h +++ b/tree.h @@ -10,14 +10,14 @@ struct strbuf; struct tree { struct object object; void *buffer; - unsigned long size; + size_t size; }; extern const char *tree_type; struct tree *lookup_tree(struct repository *r, const struct object_id *oid); -int parse_tree_buffer(struct tree *item, void *buffer, unsigned long size); +int parse_tree_buffer(struct tree *item, void *buffer, size_t size); #define parse_tree_gently(t, q) repo_parse_tree_gently(the_repository, t, q) int repo_parse_tree_gently(struct repository *r, struct tree *item, From 47bf790e0c5d6364d6b72ad82cd7b85f48db93d5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 22:12:59 +0200 Subject: [PATCH 137/229] commit: widen the commit-buffer API to size_t Continue the migration from `unsigned long` to `size_t`. The `size` attribute of `struct commit_buffer` is fed either from `odb_read_object()`'s return value (`size_t`, handled with `cast_size_t_to_ulong()`) or from `strbuf.len` in `fake_working_tree_commit()` (silently narrowed today). Widen the field and a couple of function signatures together, drop the shim in `repo_get_commit_buffer()`, and move the matching `unsigned long` locals at the in-tree callers in commit.c (three sites), builtin/replace.c, and builtin/stash.c (two sites). The remaining callers pass NULL or already pass a size_t-compatible variable. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/replace.c | 2 +- builtin/stash.c | 4 ++-- commit.c | 19 +++++++++++-------- commit.h | 8 +++++--- 4 files changed, 19 insertions(+), 14 deletions(-) diff --git a/builtin/replace.c b/builtin/replace.c index aed6b2c8debf86..b85681080dc1d9 100644 --- a/builtin/replace.c +++ b/builtin/replace.c @@ -459,7 +459,7 @@ static int create_graft(int argc, const char **argv, int force, int gentle) struct commit *commit; struct strbuf buf = STRBUF_INIT; const char *buffer; - unsigned long size; + size_t size; if (repo_get_oid(the_repository, old_ref, &old_oid) < 0) return error(_("not a valid object name: '%s'"), old_ref); diff --git a/builtin/stash.c b/builtin/stash.c index ac8fa0d42299be..c8588cdf89bf30 100644 --- a/builtin/stash.c +++ b/builtin/stash.c @@ -2045,7 +2045,7 @@ static int write_commit_with_parents(struct repository *r, const char *orig_author, *orig_committer; char *author = NULL, *committer = NULL; const char *buffer; - unsigned long bufsize; + size_t bufsize; const char *p; struct strbuf msg = STRBUF_INIT; int ret = 0; @@ -2097,7 +2097,7 @@ static int do_import_stash(struct repository *r, const char *rev) struct object_id chain; int res = 0; const char *buffer = NULL; - unsigned long bufsize; + size_t bufsize; struct commit *this = NULL; struct commit_list *items = NULL, *cur; char *msg = NULL; diff --git a/commit.c b/commit.c index ad26f0b40a9c64..9ba5af99eae693 100644 --- a/commit.c +++ b/commit.c @@ -349,7 +349,7 @@ int for_each_commit_graft(each_commit_graft_fn fn, void *cb_data) struct commit_buffer { void *buffer; - unsigned long size; + size_t size; }; define_commit_slab(buffer_slab, struct commit_buffer); @@ -366,7 +366,8 @@ void free_commit_buffer_slab(struct buffer_slab *bs) free(bs); } -void set_commit_buffer(struct repository *r, struct commit *commit, void *buffer, unsigned long size) +void set_commit_buffer(struct repository *r, struct commit *commit, + void *buffer, size_t size) { struct commit_buffer *v = buffer_slab_at( r->parsed_objects->buffer_slab, commit); @@ -374,7 +375,9 @@ void set_commit_buffer(struct repository *r, struct commit *commit, void *buffer v->size = size; } -const void *get_cached_commit_buffer(struct repository *r, const struct commit *commit, unsigned long *sizep) +const void *get_cached_commit_buffer(struct repository *r, + const struct commit *commit, + size_t *sizep) { struct commit_buffer *v = buffer_slab_peek( r->parsed_objects->buffer_slab, commit); @@ -390,7 +393,7 @@ const void *get_cached_commit_buffer(struct repository *r, const struct commit * const void *repo_get_commit_buffer(struct repository *r, const struct commit *commit, - unsigned long *sizep) + size_t *sizep) { const void *ret = get_cached_commit_buffer(r, commit, sizep); if (!ret) { @@ -404,7 +407,7 @@ const void *repo_get_commit_buffer(struct repository *r, die("expected commit for %s, got %s", oid_to_hex(&commit->object.oid), type_name(type)); if (sizep) - *sizep = cast_size_t_to_ulong(size); + *sizep = size; } return ret; } @@ -1185,7 +1188,7 @@ int parse_signed_commit(const struct commit *commit, struct strbuf *payload, struct strbuf *signature, const struct git_hash_algo *algop) { - unsigned long size; + size_t size; const char *buffer = repo_get_commit_buffer(the_repository, commit, &size); int ret = parse_buffer_signed_by_header(buffer, size, payload, signature, algop); @@ -1358,7 +1361,7 @@ int verify_commit_buffer(const char *buffer, size_t size, int check_commit_signature(const struct commit *commit, struct signature_check *sigc) { - unsigned long size; + size_t size; const char *buffer = repo_get_commit_buffer(the_repository, commit, &size); int ret = verify_commit_buffer(buffer, size, sigc); @@ -1455,7 +1458,7 @@ struct commit_extra_header *read_commit_extra_headers(struct commit *commit, const char **exclude) { struct commit_extra_header *extra = NULL; - unsigned long size; + size_t size; const char *buffer = repo_get_commit_buffer(the_repository, commit, &size); extra = read_commit_extra_header_lines(buffer, size, exclude); diff --git a/commit.h b/commit.h index 1061ed791bcad6..97779ec943809f 100644 --- a/commit.h +++ b/commit.h @@ -131,13 +131,15 @@ void free_commit_buffer_slab(struct buffer_slab *bs); * Associate an object buffer with the commit. The ownership of the * memory is handed over to the commit, and must be free()-able. */ -void set_commit_buffer(struct repository *r, struct commit *, void *buffer, unsigned long size); +void set_commit_buffer(struct repository *r, struct commit *, + void *buffer, size_t size); /* * Get any cached object buffer associated with the commit. Returns NULL * if none. The resulting memory should not be freed. */ -const void *get_cached_commit_buffer(struct repository *, const struct commit *, unsigned long *size); +const void *get_cached_commit_buffer(struct repository *, + const struct commit *, size_t *size); /* * Get the commit's object contents, either from cache or by reading the object @@ -146,7 +148,7 @@ const void *get_cached_commit_buffer(struct repository *, const struct commit *, */ const void *repo_get_commit_buffer(struct repository *r, const struct commit *, - unsigned long *size); + size_t *size); /* * Tell the commit subsystem that we are done with a particular commit buffer. From 3c3aeee3cf77b616c5408df316a42c8e3db32d7d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 09:50:48 +0200 Subject: [PATCH 138/229] pack-objects: drop cast_size_t_to_ulong shims in get_delta() The two shims that 606c192380 (odb, packfile: use size_t for streaming object sizes, 2026-05-08) and the subsequent odb_read_object() widening introduced as scaffolding around get_delta()'s reads can now disappear: the previous commit widened diff_delta() to size_t, which was the last narrow consumer in this function. Widen size and base_size to size_t outright, drop the size_st / base_size_st bridging temporaries, and drop the two cast_size_t_to_ulong() calls. Net change is 4 lines smaller and one read-then-cast indirection gone from each odb read. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/pack-objects.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index ab15529a9dabda..3824b4548e906b 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -354,21 +354,17 @@ static void index_commit_for_bitmap(struct commit *commit) static void *get_delta(struct object_entry *entry) { - unsigned long size, base_size; - size_t delta_size; + size_t size, base_size, delta_size; void *buf, *base_buf, *delta_buf; enum object_type type; - size_t size_st = 0, base_size_st = 0; buf = odb_read_object(the_repository->objects, &entry->idx.oid, - &type, &size_st); - size = cast_size_t_to_ulong(size_st); + &type, &size); if (!buf) die(_("unable to read %s"), oid_to_hex(&entry->idx.oid)); base_buf = odb_read_object(the_repository->objects, &DELTA(entry)->idx.oid, &type, - &base_size_st); - base_size = cast_size_t_to_ulong(base_size_st); + &base_size); if (!base_buf) die("unable to read %s", oid_to_hex(&DELTA(entry)->idx.oid)); From 8f523ab5d8ce73f3256e41eac37181047da692c7 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 22:19:32 +0200 Subject: [PATCH 139/229] blame: widen find_line_starts() len parameter to size_t Prep for the upcoming blame_scoreboard.final_buf_size widening: prepare_lines() will pass a size_t through to find_line_starts(), and the other caller (fill_origin_blob() via o->file.size) already goes through long->size_t promotion. The function is file-static and only uses len as a loop bound. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- blame.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blame.c b/blame.c index 3189bcdabaa886..f8b3e4e90ed05d 100644 --- a/blame.c +++ b/blame.c @@ -337,7 +337,7 @@ static const char *get_next_line(const char *start, const char *end) } static int find_line_starts(int **line_starts, const char *buf, - unsigned long len) + size_t len) { const char *end = buf + len; const char *p; From 782e87de7e101688498076b6e9ad50e1b512b644 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 10:49:54 +0200 Subject: [PATCH 140/229] pack-objects: drop cast_size_t_to_ulong shims in try_delta() Companion to the prior get_delta() cleanup, and the last try_delta() piece of the >4 GiB delta-path topic. Every consumer that the function's locals fed has now been widened: SIZE() / DELTA_SIZE() to size_t (prior topic), the mem_usage out-parameter and delta_cacheable() earlier in this series, and create_delta() / create_delta_index() in the immediately preceding commits. Widen the declaration of trg_size, src_size, sizediff, max_size and sz to size_t (delta_size joins them on the same line, removing the size_t delta_size line that the create_delta() widening commit added as a stop-gap), and drop the two sz_st bridge variables together with the surrounding cast_size_t_to_ulong() calls. The result is just "odb_read_object(&sz)" on both reads. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/pack-objects.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 3824b4548e906b..57931664324755 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -2796,8 +2796,7 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, { struct object_entry *trg_entry = trg->entry; struct object_entry *src_entry = src->entry; - unsigned long trg_size, src_size, sizediff, max_size, sz; - size_t delta_size; + size_t trg_size, src_size, delta_size, sizediff, max_size, sz; unsigned ref_depth; enum object_type type; void *delta_buf; @@ -2850,12 +2849,10 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, /* Load data if not already done */ if (!trg->data) { - size_t sz_st = 0; packing_data_lock(&to_pack); trg->data = odb_read_object(the_repository->objects, &trg_entry->idx.oid, &type, - &sz_st); - sz = cast_size_t_to_ulong(sz_st); + &sz); packing_data_unlock(&to_pack); if (!trg->data) die(_("object %s cannot be read"), @@ -2867,12 +2864,10 @@ static int try_delta(struct unpacked *trg, struct unpacked *src, *mem_usage += sz; } if (!src->data) { - size_t sz_st = 0; packing_data_lock(&to_pack); src->data = odb_read_object(the_repository->objects, &src_entry->idx.oid, &type, - &sz_st); - sz = cast_size_t_to_ulong(sz_st); + &sz); packing_data_unlock(&to_pack); if (!src->data) { if (src_entry->preferred_base) { From 7049c8eafa0ba769a99a7200b360d870ddc49722 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 22:30:31 +0200 Subject: [PATCH 141/229] grep: widen struct grep_source.size and grep_buffer() to size_t This commit continue the migration from `unsigned long` to `size_t`, converting `grep_buffer()` and helpers. The callers are already prepared for this change. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- grep.c | 18 ++++++++---------- grep.h | 4 ++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/grep.c b/grep.c index 1d75d314211aa8..d75fbcef449f7e 100644 --- a/grep.c +++ b/grep.c @@ -864,9 +864,9 @@ void free_grep_patterns(struct grep_opt *opt) free_pattern_expr(opt->pattern_expression); } -static const char *end_of_line(const char *cp, unsigned long *left) +static const char *end_of_line(const char *cp, size_t *left) { - unsigned long l = *left; + size_t l = *left; while (l && *cp != '\n') { l--; cp++; @@ -1454,7 +1454,7 @@ static int should_lookahead(struct grep_opt *opt) } static int look_ahead(struct grep_opt *opt, - unsigned long *left_p, + size_t *left_p, unsigned *lno_p, const char **bol_p) { @@ -1567,7 +1567,7 @@ static int grep_source_1(struct grep_opt *opt, struct grep_source *gs, int colle { const char *bol; const char *peek_bol = NULL; - unsigned long left; + size_t left; unsigned lno = 1; unsigned last_hit = 0; int binary_match_only = 0; @@ -1735,7 +1735,7 @@ static int grep_source_1(struct grep_opt *opt, struct grep_source *gs, int colle goto next_line; } if (show_function && (!peek_bol || peek_bol < bol)) { - unsigned long peek_left = left; + size_t peek_left = left; const char *peek_eol = eol; /* @@ -1854,7 +1854,7 @@ int grep_source(struct grep_opt *opt, struct grep_source *gs) static void grep_source_init_buf(struct grep_source *gs, const char *buf, - unsigned long size) + size_t size) { gs->type = GREP_SOURCE_BUF; gs->name = NULL; @@ -1865,7 +1865,7 @@ static void grep_source_init_buf(struct grep_source *gs, gs->identifier = NULL; } -int grep_buffer(struct grep_opt *opt, const char *buf, unsigned long size) +int grep_buffer(struct grep_opt *opt, const char *buf, size_t size) { struct grep_source gs; int r; @@ -1931,11 +1931,9 @@ void grep_source_clear_data(struct grep_source *gs) static int grep_source_load_oid(struct grep_source *gs) { enum object_type type; - size_t size_st = 0; gs->buf = odb_read_object(gs->repo->objects, gs->identifier, - &type, &size_st); - gs->size = cast_size_t_to_ulong(size_st); + &type, &gs->size); if (!gs->buf) return error(_("'%s': unable to read %s"), gs->name, diff --git a/grep.h b/grep.h index 13e26a93189c3b..0bd705dfc0b82b 100644 --- a/grep.h +++ b/grep.h @@ -212,7 +212,7 @@ void append_grep_pattern(struct grep_opt *opt, const char *pat, const char *orig void append_header_grep_pattern(struct grep_opt *, enum grep_header_field, const char *); void compile_grep_patterns(struct grep_opt *opt); void free_grep_patterns(struct grep_opt *opt); -int grep_buffer(struct grep_opt *opt, const char *buf, unsigned long size); +int grep_buffer(struct grep_opt *opt, const char *buf, size_t size); /* The field parameter is only used to filter header patterns * (where appropriate). If filtering isn't desirable @@ -235,7 +235,7 @@ struct grep_source { struct repository *repo; /* if GREP_SOURCE_OID */ const char *buf; - unsigned long size; + size_t size; char *path; /* for attribute lookups */ struct userdiff_driver *driver; From 75c17b1e488a3af5e4fd9d4c7f7c645dd4eaf34b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 14:39:26 +0200 Subject: [PATCH 142/229] pack-objects: drop the last size shim in write_no_reuse_object() Continue the size_t evacuation that this series and the merged js/objects-larger-than-4gb-on-windows topic are advancing for >4 GiB objects on Windows: with the odb readers and the zlib helpers reached from do_compress() now widened end-to-end, the last cast_size_t_to_ulong() shim in this function can be removed, and do_compress() itself can carry the new size type through. Two cast_size_t_to_ulong() shims remain in this file; they feed the tree-walk API, which is still narrow and is a separate widening topic. write_no_reuse_object()'s return type and the hashfile API are still narrow but unchanged in observable behaviour: on 64-bit Linux ulong coincides with size_t, and on Windows these were the narrow fenceposts the prior topics deliberately left in place. Their widening is left to follow-ups touching the hashfile API and the write_object() caller chain. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/pack-objects.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 57931664324755..160fcb376265bb 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -382,11 +382,11 @@ static void *get_delta(struct object_entry *entry) return delta_buf; } -static unsigned long do_compress(void **pptr, unsigned long size) +static size_t do_compress(void **pptr, size_t size) { git_zstream stream; void *in, *out; - unsigned long maxsize; + size_t maxsize; struct repo_config_values *cfg = repo_config_values(the_repository); git_deflate_init(&stream, cfg->pack_compression_level); @@ -512,7 +512,7 @@ static inline int oe_size_greater_than(struct packing_data *pack, static unsigned long write_no_reuse_object(struct hashfile *f, struct object_entry *entry, unsigned long limit, int usable_delta) { - unsigned long size, datalen; + size_t size, datalen; unsigned char header[MAX_PACK_OBJECT_HEADER], dheader[MAX_PACK_OBJECT_HEADER]; unsigned hdrlen; @@ -531,11 +531,9 @@ static unsigned long write_no_reuse_object(struct hashfile *f, struct object_ent type = st->type; size = st->size; } else { - size_t size_st = 0; buf = odb_read_object(the_repository->objects, &entry->idx.oid, &type, - &size_st); - size = cast_size_t_to_ulong(size_st); + &size); if (!buf) die(_("unable to read %s"), oid_to_hex(&entry->idx.oid)); From 909d4b76e29ba5ed164065aef00e8cd423dbf2a1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 22:41:07 +0200 Subject: [PATCH 143/229] fast-export: drop the export_blob() size cast and widen anonymize_blob() Mirror of the preceding fast-import sweep. anonymize_blob() writes strbuf.len (size_t) into its out-parameter, and export_blob()'s non-anonymize branch reads odb_read_object()'s size_t out-parameter through a size_st + cast_size_t_to_ulong() bridge into an unsigned long local; both have been silent on Windows past 4 GiB. Widen the helper signature and the local, and drop the bridge. check_object_signature() and parse_object_buffer() still take unsigned long, so the silent narrowing on Windows just moves from the local assignment to those call sites; both are separate topics. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/fast-export.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/builtin/fast-export.c b/builtin/fast-export.c index 50278bef7a6a4d..cd7705c2fb8588 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -285,7 +285,7 @@ static void show_progress(void) * There's no need to cache this result with anonymize_mem, since * we already handle blob content caching with marks. */ -static char *anonymize_blob(unsigned long *size) +static char *anonymize_blob(size_t *size) { static int counter; struct strbuf out = STRBUF_INIT; @@ -296,7 +296,7 @@ static char *anonymize_blob(unsigned long *size) static void export_blob(const struct object_id *oid) { - unsigned long size; + size_t size; enum object_type type; char *buf; struct object *object; @@ -317,10 +317,8 @@ static void export_blob(const struct object_id *oid) object = (struct object *)lookup_blob(the_repository, oid); eaten = 0; } else { - size_t size_st = 0; buf = odb_read_object(the_repository->objects, oid, &type, - &size_st); - size = cast_size_t_to_ulong(size_st); + &size); if (!buf) die(_("could not read blob %s"), oid_to_hex(oid)); if (check_object_signature(the_repository, oid, buf, size, From 712d4fb9f4208e00aea1b1d4b759429ae61b69e6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 22:23:09 +0200 Subject: [PATCH 144/229] blame: widen struct blame_scoreboard.final_buf_size to size_t Continue the size_t evacuation. final_buf_size is fed either from textconv_object()'s now-size_t out-parameter, from odb_read_object()'s size_t out-parameter (both bridged today through a final_buf_size_st local + cast_size_t_to_ulong()), or from o->file.size (mmfile_t, long). Widen the struct field, point both producers straight at it, and drop the bridge variable along with the cast. builtin/blame.c only reads the field for pointer arithmetic and comparisons, which promote cleanly. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- blame.c | 6 ++---- blame.h | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/blame.c b/blame.c index 0c8fa69a623c39..2707078a77f9a8 100644 --- a/blame.c +++ b/blame.c @@ -2928,20 +2928,18 @@ void setup_scoreboard(struct blame_scoreboard *sb, sb->final_buf_size = o->file.size; } else { - size_t final_buf_size_st = 0; o = get_origin(sb->final, sb->path); if (fill_blob_sha1_and_mode(sb->repo, o)) die(_("no such path %s in %s"), sb->path, final_commit_name); if (sb->revs->diffopt.flags.allow_textconv && textconv_object(sb->repo, sb->path, o->mode, &o->blob_oid, 1, (char **) &sb->final_buf, - &final_buf_size_st)) + &sb->final_buf_size)) ; else sb->final_buf = odb_read_object(the_repository->objects, &o->blob_oid, &type, - &final_buf_size_st); - sb->final_buf_size = cast_size_t_to_ulong(final_buf_size_st); + &sb->final_buf_size); if (!sb->final_buf) die(_("cannot read blob %s for path %s"), diff --git a/blame.h b/blame.h index 3b34be0e5c6932..1b66084a8d687a 100644 --- a/blame.h +++ b/blame.h @@ -117,7 +117,7 @@ struct blame_scoreboard { * indexed with scoreboard.lineno[blame_entry.lno]. */ char *final_buf; - unsigned long final_buf_size; + size_t final_buf_size; /* linked list of blames */ struct blame_entry *ent; From 8aac8238c176c207d92385199cd87c0cd960501e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 22:46:50 +0200 Subject: [PATCH 145/229] repo: drop the inflated-size cast in count_objects() Continue the size_t evacuation. count_objects() feeds the inflated size from odb_read_object_info_extended()'s size_t out-parameter into struct object_values (size_t) and check_largest() (size_t) through an unsigned long bridge with a cast_size_t_to_ulong() shim. The bridge was the only narrow link in the chain. Widen the local, point oi.sizep at it directly, and drop the cast. parse_object_buffer() still takes unsigned long, so a Windows narrowing remains at that one call; that is its own follow-up topic. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/repo.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/builtin/repo.c b/builtin/repo.c index 6414a91a1d54b7..5f1064af78175f 100644 --- a/builtin/repo.c +++ b/builtin/repo.c @@ -986,15 +986,14 @@ static int count_objects(const char *path UNUSED, struct oid_array *oids, for (size_t i = 0; i < oids->nr; i++) { struct object_info oi = OBJECT_INFO_INIT; - unsigned long inflated; - size_t inflated_st = 0; + size_t inflated; struct commit *commit; struct object *obj; void *content; off_t disk; int eaten; - oi.sizep = &inflated_st; + oi.sizep = &inflated; oi.disk_sizep = &disk; oi.contentp = &content; @@ -1002,7 +1001,6 @@ static int count_objects(const char *path UNUSED, struct oid_array *oids, OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK) < 0) continue; - inflated = cast_size_t_to_ulong(inflated_st); obj = parse_object_buffer(the_repository, &oids->oid[i], type, inflated, content, &eaten); From bf3797552286acbc0839fc9ec21b4308678b9cd0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 22:37:32 +0200 Subject: [PATCH 146/229] fast-import: drop the six size casts in the object-read paths Continue the size_t evacuation. fast-import's helper gfi_unpack_entry() and the five size-handling sites that feed off it (store_object()'s deltalen, load_tree(), parse_from_existing(), the inline gfi_unpack_entry() caller in parse_objectish(), cat_blob(), and dereference()) all carry size_t-shaped values from the odb / unpack_entry() APIs through cast_size_t_to_ulong() bridges into unsigned long locals. With the producers (odb_read_object(), odb_read_object_peeled(), unpack_entry()) and the consumers it feeds (the zlib avail_in field from a prior commit, encode_in_pack_object_header()'s uintmax_t parameter, parse_from_commit()'s widened size parameter) all size_t-ready, the bridges and casts go away in one pass. gfi_unpack_entry() now writes into the caller's size_t directly, and the six locals collapse to plain size_t declarations. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/fast-import.c | 42 ++++++++++++++---------------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 7b1a9d1e09a9d3..3564666ef9014b 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -980,7 +980,7 @@ static int store_object( struct object_entry *e; unsigned char hdr[96]; struct object_id oid; - unsigned long hdrlen, deltalen = 0; + size_t hdrlen, deltalen = 0; struct git_hash_ctx c; git_zstream s; struct repo_config_values *cfg = repo_config_values(the_repository); @@ -1014,13 +1014,10 @@ static int store_object( if (last && last->data.len && last->data.buf && last->depth < max_depth && dat->len > the_hash_algo->rawsz) { - size_t deltalen_st; - delta_count_attempts_by_type[type]++; delta = diff_delta(last->data.buf, last->data.len, dat->buf, dat->len, - &deltalen_st, dat->len - the_hash_algo->rawsz); - deltalen = cast_size_t_to_ulong(deltalen_st); + &deltalen, dat->len - the_hash_algo->rawsz); } else delta = NULL; @@ -1255,10 +1252,9 @@ static void stream_blob(uintmax_t len, struct object_id *oidout, uintmax_t mark) */ static void *gfi_unpack_entry( struct object_entry *oe, - unsigned long *sizep) + size_t *sizep) { enum object_type type; - size_t size_st = 0; void *data; struct packed_git *p = all_packs[oe->pack_id]; if (p == pack_data && p->pack_size < (pack_size + the_hash_algo->rawsz)) { @@ -1281,9 +1277,7 @@ static void *gfi_unpack_entry( */ p->pack_size = pack_size + the_hash_algo->rawsz; } - data = unpack_entry(the_repository, p, oe->idx.offset, &type, &size_st); - if (sizep) - *sizep = cast_size_t_to_ulong(size_st); + data = unpack_entry(the_repository, p, oe->idx.offset, &type, sizep); return data; } @@ -1292,7 +1286,7 @@ static void load_tree(struct tree_entry *root) struct object_id *oid = &root->versions[1].oid; struct object_entry *myoe; struct tree_content *t; - unsigned long size; + size_t size; char *buf; const char *c; @@ -1310,10 +1304,8 @@ static void load_tree(struct tree_entry *root) die(_("can't load tree %s"), oid_to_hex(oid)); } else { enum object_type type; - size_t size_st = 0; buf = odb_read_object(the_repository->objects, oid, &type, - &size_st); - size = cast_size_t_to_ulong(size_st); + &size); if (!buf || type != OBJ_TREE) die(_("can't load tree %s"), oid_to_hex(oid)); } @@ -2636,7 +2628,7 @@ static void file_change_deleteall(struct branch *b) b->num_notes = 0; } -static void parse_from_commit(struct branch *b, char *buf, unsigned long size) +static void parse_from_commit(struct branch *b, char *buf, size_t size) { if (!buf || size < the_hash_algo->hexsz + 6) die(_("not a valid commit: %s"), oid_to_hex(&b->oid)); @@ -2653,13 +2645,11 @@ static void parse_from_existing(struct branch *b) oidclr(&b->branch_tree.versions[0].oid, the_repository->hash_algo); oidclr(&b->branch_tree.versions[1].oid, the_repository->hash_algo); } else { - unsigned long size; - size_t size_st = 0; + size_t size; char *buf; buf = odb_read_object_peeled(the_repository->objects, &b->oid, - OBJ_COMMIT, &size_st, &b->oid); - size = cast_size_t_to_ulong(size_st); + OBJ_COMMIT, &size, &b->oid); parse_from_commit(b, buf, size); free(buf); } @@ -2690,7 +2680,7 @@ static int parse_objectish(struct fast_import_state *state, if (!oideq(&b->oid, &oe->idx.oid)) { oidcpy(&b->oid, &oe->idx.oid); if (oe->pack_id != MAX_PACK_ID) { - unsigned long size; + size_t size; char *buf = gfi_unpack_entry(oe, &size); parse_from_commit(b, buf, size); free(buf); @@ -3363,15 +3353,13 @@ static void cat_blob(struct object_entry *oe, struct object_id *oid) { struct strbuf line = STRBUF_INIT; struct iovec iov[3]; - unsigned long size; + size_t size; enum object_type type = 0; char *buf; if (!oe || oe->pack_id == MAX_PACK_ID) { - size_t size_st = 0; buf = odb_read_object(the_repository->objects, oid, &type, - &size_st); - size = cast_size_t_to_ulong(size_st); + &size); } else { type = oe->type; buf = gfi_unpack_entry(oe, &size); @@ -3463,7 +3451,7 @@ static void parse_cat_blob(struct fast_import_state *state UNUSED, static struct object_entry *dereference(struct object_entry *oe, struct object_id *oid) { - unsigned long size; + size_t size; char *buf = NULL; const unsigned hexsz = the_hash_algo->hexsz; @@ -3492,10 +3480,8 @@ static struct object_entry *dereference(struct object_entry *oe, buf = gfi_unpack_entry(oe, &size); } else { enum object_type unused; - size_t size_st = 0; buf = odb_read_object(the_repository->objects, oid, - &unused, &size_st); - size = cast_size_t_to_ulong(size_st); + &unused, &size); } if (!buf) die(_("can't load object %s"), oid_to_hex(oid)); From d9089faaefc5c99a1ae0135c690f3556ff776480 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 23:05:52 +0200 Subject: [PATCH 147/229] unpack-objects: widen the size-passing infrastructure to size_t Drop the last cast_size_t_to_ulong() in builtin/unpack-objects.c. With size_t-typed object sizes already coming in via odb_read_object() and the per-byte varint decode in unpack_one() (widened by f2063855fb), the rest of the file was the only thing left that still threaded sizes through unsigned long: struct obj_buffer.size and struct delta_info.size, get_data() and add_object_buffer(), add_delta_to_list(), resolve_delta(), resolve_against_held(), added_object(), write_object(), unpack_non_delta_entry(), unpack_delta_entry(), and stream_blob(). Widen all of them together. None of those types had a downstream narrow consumer once odb_write_object() and patch_delta() were widened earlier, so the change is mechanical: parameter and field types change, the base_size_st bridge in unpack_delta_entry() and its cast go away, and odb_read_object() now writes into base_size directly. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- builtin/unpack-objects.c | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 351948724ab703..e0183b7a73e820 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -40,7 +40,7 @@ static struct progress *progress; */ struct obj_buffer { char *buffer; - unsigned long size; + size_t size; }; static struct decoration obj_decorate; @@ -50,7 +50,7 @@ static struct obj_buffer *lookup_object_buffer(struct object *base) return lookup_decoration(&obj_decorate, base); } -static void add_object_buffer(struct object *object, char *buffer, unsigned long size) +static void add_object_buffer(struct object *object, char *buffer, size_t size) { struct obj_buffer *obj; CALLOC_ARRAY(obj, 1); @@ -114,10 +114,10 @@ static void use(int bytes) * allocated buffer which is reused to hold temporary zstream output * and return NULL instead of returning garbage data. */ -static void *get_data(unsigned long size) +static void *get_data(size_t size) { git_zstream stream; - unsigned long bufsize = dry_run && size > 8192 ? 8192 : size; + size_t bufsize = dry_run && size > 8192 ? 8192 : size; void *buf = xmallocz(bufsize); memset(&stream, 0, sizeof(stream)); @@ -161,7 +161,7 @@ struct delta_info { struct object_id base_oid; unsigned nr; off_t base_offset; - unsigned long size; + size_t size; void *delta; struct delta_info *next; }; @@ -170,7 +170,7 @@ static struct delta_info *delta_list; static void add_delta_to_list(unsigned nr, const struct object_id *base_oid, off_t base_offset, - void *delta, unsigned long size) + void *delta, size_t size) { struct delta_info *info = xmalloc(sizeof(*info)); @@ -261,7 +261,7 @@ static void write_rest(void) } static void added_object(unsigned nr, enum object_type type, - void *data, unsigned long size); + void *data, size_t size); /* * Write out nr-th object from the list, now we know the contents @@ -269,7 +269,7 @@ static void added_object(unsigned nr, enum object_type type, * to be checked at the end. */ static void write_object(unsigned nr, enum object_type type, - void *buf, unsigned long size) + void *buf, size_t size) { if (!strict) { if (odb_write_object(the_repository->objects, buf, size, type, @@ -310,8 +310,8 @@ static void write_object(unsigned nr, enum object_type type, } static void resolve_delta(unsigned nr, enum object_type type, - void *base, unsigned long base_size, - void *delta, unsigned long delta_size) + void *base, size_t base_size, + void *delta, size_t delta_size) { void *result; size_t result_size; @@ -330,7 +330,7 @@ static void resolve_delta(unsigned nr, enum object_type type, * resolve all the deltified objects that are based on it. */ static void added_object(unsigned nr, enum object_type type, - void *data, unsigned long size) + void *data, size_t size) { struct delta_info **p = &delta_list; struct delta_info *info; @@ -349,7 +349,7 @@ static void added_object(unsigned nr, enum object_type type, } } -static void unpack_non_delta_entry(enum object_type type, unsigned long size, +static void unpack_non_delta_entry(enum object_type type, size_t size, unsigned nr) { void *buf = get_data(size); @@ -386,7 +386,7 @@ static ssize_t zlib_stream_read(struct odb_stream *in_stream, return buf_len - zstream->avail_out; } -static void stream_blob(unsigned long size, unsigned nr) +static void stream_blob(size_t size, unsigned nr) { git_zstream zstream = { 0 }; struct zlib_stream in_stream = { @@ -420,7 +420,7 @@ static void stream_blob(unsigned long size, unsigned nr) } static int resolve_against_held(unsigned nr, const struct object_id *base, - void *delta_data, unsigned long delta_size) + void *delta_data, size_t delta_size) { struct object *obj; struct obj_buffer *obj_buffer; @@ -435,12 +435,11 @@ static int resolve_against_held(unsigned nr, const struct object_id *base, return 1; } -static void unpack_delta_entry(enum object_type type, unsigned long delta_size, +static void unpack_delta_entry(enum object_type type, size_t delta_size, unsigned nr) { void *delta_data, *base; - unsigned long base_size; - size_t base_size_st = 0; + size_t base_size; struct object_id base_oid; if (type == OBJ_REF_DELTA) { @@ -517,8 +516,7 @@ static void unpack_delta_entry(enum object_type type, unsigned long delta_size, return; base = odb_read_object(the_repository->objects, &base_oid, - &type, &base_size_st); - base_size = cast_size_t_to_ulong(base_size_st); + &type, &base_size); if (!base) { error("failed to read delta-pack base object %s", oid_to_hex(&base_oid)); From 14dadb738310ac921b0971c445bcc6540c2b0a4f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 30 Jul 2026 14:19:17 +0200 Subject: [PATCH 148/229] mingw: handle staging 4GB+ files correctly This is yet another set of `unsigned long` declarations that should have been `size_t` ones. Signed-off-by: Johannes Schindelin --- odb.c | 2 +- odb.h | 4 ++-- odb/source-loose.c | 2 +- odb/source.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/odb.c b/odb.c index d1a9b37880b890..b84408b0905bff 100644 --- a/odb.c +++ b/odb.c @@ -668,7 +668,7 @@ void odb_assert_oid_type(struct object_database *odb, } int odb_write_object_ext(struct object_database *odb, - const void *buf, unsigned long len, + const void *buf, size_t len, enum object_type type, struct object_id *oid, const struct object_id *compat_oid_in, diff --git a/odb.h b/odb.h index 4008f41308ba5b..766883f2e14c0e 100644 --- a/odb.h +++ b/odb.h @@ -637,14 +637,14 @@ enum odb_write_object_flags { * Returns 0 on success, a negative error code otherwise. */ int odb_write_object_ext(struct object_database *odb, - const void *buf, unsigned long len, + const void *buf, size_t len, enum object_type type, struct object_id *oid, const struct object_id *compat_oid, enum odb_write_object_flags flags); static inline int odb_write_object(struct object_database *odb, - const void *buf, unsigned long len, + const void *buf, size_t len, enum object_type type, struct object_id *oid) { diff --git a/odb/source-loose.c b/odb/source-loose.c index 0664634938aef3..fe1a2983645a3c 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -770,7 +770,7 @@ static int end_loose_object_common(struct odb_source_loose *loose, static int write_loose_object(struct odb_source_loose *loose, const struct object_id *oid, char *hdr, - int hdrlen, const void *buf, unsigned long len, + int hdrlen, const void *buf, size_t len, const time_t *mtime, unsigned flags) { int fd, ret; diff --git a/odb/source.h b/odb/source.h index 1867347dfcff7a..8e6ed96399f020 100644 --- a/odb/source.h +++ b/odb/source.h @@ -469,7 +469,7 @@ static inline int odb_source_freshen_object(struct odb_source *source, * object ID and the compatibility object ID, if non-NULL. */ static inline int odb_source_write_object(struct odb_source *source, - const void *buf, unsigned long len, + const void *buf, size_t len, enum object_type type, const struct object_id *oid, const struct object_id *compat_oid, From e661a01945aa5d44681c3eee90f488dc2e402248 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 5 Jun 2026 22:50:00 +0200 Subject: [PATCH 149/229] t/helper/test-pack-deltas: drop the delta_size cast in write_ref_delta() Tidies up the bridge variable introduced in the create_delta() / diff_delta() widening commit earlier in this series. With the test helper's local do_compress() also widened to size_t in pass, the narrowing into the unsigned long delta_size local that compress expected is gone, the size_st bridge is unnecessary, and the cast goes away. encode_in_pack_object_header() takes uintmax_t and hashwrite() takes uint32_t, both unchanged. Assisted-by: Opus 4.7 Signed-off-by: Johannes Schindelin --- t/helper/test-pack-deltas.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/t/helper/test-pack-deltas.c b/t/helper/test-pack-deltas.c index 959705fecaf144..8b448485fbf4ca 100644 --- a/t/helper/test-pack-deltas.c +++ b/t/helper/test-pack-deltas.c @@ -18,7 +18,7 @@ static const char *usage_str[] = { NULL }; -static unsigned long do_compress(void **pptr, unsigned long size) +static size_t do_compress(void **pptr, size_t size) { git_zstream stream; void *in, *out; @@ -48,8 +48,8 @@ static void write_ref_delta(struct hashfile *f, struct object_id *base) { unsigned char header[MAX_PACK_OBJECT_HEADER]; - unsigned long delta_size, compressed_size, hdrlen; - size_t size, base_size, delta_size_st = 0; + unsigned long compressed_size, hdrlen; + size_t size, base_size, delta_size; enum object_type type; void *base_buf, *delta_buf; void *buf = odb_read_object(the_repository->objects, @@ -65,8 +65,7 @@ static void write_ref_delta(struct hashfile *f, die("unable to read %s", oid_to_hex(base)); delta_buf = diff_delta(base_buf, base_size, - buf, size, &delta_size_st, 0); - delta_size = cast_size_t_to_ulong(delta_size_st); + buf, size, &delta_size, 0); compressed_size = do_compress(&delta_buf, delta_size); From f1ac9e1af1747ce774534b79e489910daa7f3096 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 30 Jul 2026 14:42:25 +0200 Subject: [PATCH 150/229] mingw: support unpacking loose 4GB+ objects ... and yet another round of `unsigned long` -> `size_t` transmogrifications. Signed-off-by: Johannes Schindelin --- object-file.c | 10 +++++----- object-file.h | 6 +++--- odb/source-loose.c | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/object-file.c b/object-file.c index 2db9ca32217b3d..bba03a683dbae3 100644 --- a/object-file.c +++ b/object-file.c @@ -155,7 +155,7 @@ int stream_object_signature(struct repository *r, * Map and close the given loose object fd. The path argument is used for * error reporting. */ -static void *map_fd(int fd, const char *path, unsigned long *size) +static void *map_fd(int fd, const char *path, size_t *size) { void *map = NULL; struct stat st; @@ -176,9 +176,9 @@ static void *map_fd(int fd, const char *path, unsigned long *size) enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, unsigned char *map, - unsigned long mapsize, + size_t mapsize, void *buffer, - unsigned long bufsiz) + size_t bufsiz) { int status; @@ -209,7 +209,7 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, } void *unpack_loose_rest(git_zstream *stream, - void *buffer, unsigned long size, + void *buffer, size_t size, const struct object_id *oid) { size_t bytes = strlen(buffer) + 1, n; @@ -1196,7 +1196,7 @@ int read_loose_object(struct repository *repo, int ret = -1; int fd; void *map = NULL; - unsigned long mapsize; + size_t mapsize; git_zstream stream; char hdr[MAX_HEADER_LEN]; size_t *size = oi->sizep; diff --git a/object-file.h b/object-file.h index f44758c4f8ba01..a8da9dcd22f918 100644 --- a/object-file.h +++ b/object-file.h @@ -157,11 +157,11 @@ enum unpack_loose_header_result { */ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, unsigned char *map, - unsigned long mapsize, + size_t mapsize, void *buffer, - unsigned long bufsiz); + size_t bufsiz); void *unpack_loose_rest(git_zstream *stream, - void *buffer, unsigned long size, + void *buffer, size_t size, const struct object_id *oid); int parse_loose_header(const char *hdr, struct object_info *oi); diff --git a/odb/source-loose.c b/odb/source-loose.c index fe1a2983645a3c..5e07e10f62c53f 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -73,7 +73,7 @@ static int read_object_info_from_path(struct odb_source_loose *loose, { int ret; int fd; - unsigned long mapsize; + size_t mapsize; void *map = NULL; git_zstream stream, *stream_to_end = NULL; char hdr[MAX_HEADER_LEN]; @@ -260,7 +260,7 @@ static int open_loose_object(struct odb_source_loose *loose, static void *odb_source_loose_map_object(struct odb_source_loose *loose, const struct object_id *oid, - unsigned long *size) + size_t *size) { const char *p; int fd = open_loose_object(loose, oid, &p); @@ -295,7 +295,7 @@ struct odb_loose_read_stream { ODB_LOOSE_READ_STREAM_ERROR, } z_state; void *mapped; - unsigned long mapsize; + size_t mapsize; char hdr[32]; int hdr_avail; int hdr_used; @@ -366,7 +366,7 @@ static int odb_source_loose_read_object_stream(struct odb_stream **out, struct odb_source_loose *loose = odb_source_loose_downcast(source); struct object_info oi = OBJECT_INFO_INIT; struct odb_loose_read_stream *st; - unsigned long mapsize; + size_t mapsize; void *mapped; mapped = odb_source_loose_map_object(loose, oid, &mapsize); From b3145d1b215a345a40268daede4a6b48d62a0953 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 22 Jun 2026 11:01:41 +0200 Subject: [PATCH 151/229] Drop the `cast_size_t_to_ulong()` helper Now that all of the call sites of this helper (which I used as a kind of "NEEDSWORK" marker) are eliminated, we can drop that helper altogether. Signed-off-by: Johannes Schindelin --- git-compat-util.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/git-compat-util.h b/git-compat-util.h index a0f901ce793ed6..682fa26475303a 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -670,15 +670,6 @@ static inline size_t st_left_shift(size_t a, unsigned shift) return a << shift; } -static inline unsigned long cast_size_t_to_ulong(size_t a) -{ - if (a != (unsigned long)a) - die("object too large to read on this platform: %" - PRIuMAX" is cut off to %lu", - (uintmax_t)a, (unsigned long)a); - return (unsigned long)a; -} - static inline uint32_t cast_size_t_to_uint32_t(size_t a) { if (a != (uint32_t)a) From 3e40c82eb0c22783722c7d0e9ea661daf4e2bc16 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 30 Jul 2026 14:57:54 +0200 Subject: [PATCH 152/229] t: add an expensive test to verify that 4GB+ blobs can be staged/read I should have added this test as soon as https://github.com/git-for-windows/git/issues/6012 was opened. Signed-off-by: Johannes Schindelin --- t/meson.build | 1 + t/t3706-add-4gb-file.sh | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100755 t/t3706-add-4gb-file.sh diff --git a/t/meson.build b/t/meson.build index ed252313c96c74..0e6680eda30a61 100644 --- a/t/meson.build +++ b/t/meson.build @@ -439,6 +439,7 @@ integration_tests = [ 't3703-add-magic-pathspec.sh', 't3704-add-pathspec-file.sh', 't3705-add-sparse-checkout.sh', + 't3706-add-4gb-file.sh', 't3800-mktag.sh', 't3900-i18n-commit.sh', 't3901-i18n-patch.sh', diff --git a/t/t3706-add-4gb-file.sh b/t/t3706-add-4gb-file.sh new file mode 100755 index 00000000000000..5978ca3ee24c8a --- /dev/null +++ b/t/t3706-add-4gb-file.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# +# Copyright (c) 2026 Johannes Schindelin +# + +test_description='add 4GB+ objects' + +. ./test-lib.sh + +if ! test_have_prereq EXPENSIVE,SIZE_T_IS_64BIT +then + skip_all='expensive 4GB blob test; enable on 64-bit with GIT_TEST_LONG=true' + test_done +fi + +size_4gb=4294967296 + +test_expect_success 'set up a 4GB file' ' + test_atexit "rm -f large" && + # genrandom takes only an unsigned long... + test-tool genrandom 123 $(($size_4gb-1)) >large && + printf 1 >>large +' + +test_expect_success 'add 4GB file' ' + git add large && + git cat-file -s :large >size-staged && + test $size_4gb = $(cat size-staged) +' +test_expect_success 'read 4GB loose object' ' + git -P show :large >read && + test_file_size read >size-read && + test $size_4gb = $(cat size-read) +' + +test_done From 812121cc80b99786c5bc2eca959a3e0f2e9c1373 Mon Sep 17 00:00:00 2001 From: David PRUDHOMME Date: Tue, 15 Sep 2026 20:17:00 +0200 Subject: [PATCH 153/229] object-file: use size_t for object size in check_object_signature() Signed-off-by: David PRUDHOMME --- object-file.c | 2 +- object-file.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/object-file.c b/object-file.c index bba03a683dbae3..ddfcfbe90d34ae 100644 --- a/object-file.c +++ b/object-file.c @@ -111,7 +111,7 @@ int format_object_header(char *str, size_t size, enum object_type type, } int check_object_signature(struct repository *r, const struct object_id *oid, - void *buf, unsigned long size, + void *buf, size_t size, enum object_type type) { const struct git_hash_algo *algo = diff --git a/object-file.h b/object-file.h index a8da9dcd22f918..ee3aac7965a527 100644 --- a/object-file.h +++ b/object-file.h @@ -92,7 +92,7 @@ int format_object_header(char *str, size_t size, enum object_type type, * what we expected, but it might also indicate another error. */ int check_object_signature(struct repository *r, const struct object_id *oid, - void *map, unsigned long size, + void *map, size_t size, enum object_type type); /** From d2f073aa25e506cb480b88ca1d9fae92fd788188 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 23 Jun 2026 14:50:25 +0200 Subject: [PATCH 154/229] coverity: skip building with Rust, for now CI runs in GitHub Actions runners are ill-equipped to build with Rust, as the Windows/GCC-compatible toolchain isn't set up. Signed-off-by: Johannes Schindelin --- .github/workflows/coverity.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 58a78f1eb3f836..69d00fb9700ac7 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -37,6 +37,7 @@ jobs: COVERITY_PROJECT: ${{ vars.COVERITY_PROJECT || 'git' }} COVERITY_LANGUAGE: cxx COVERITY_PLATFORM: overridden-below + NO_RUST: Yup steps: - uses: actions/checkout@v6 - name: install minimal Git for Windows SDK From 2704ef78e8ac63a3998c74a15ac2efe0d06f5cb4 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sat, 6 Jul 2013 02:09:35 +0200 Subject: [PATCH 155/229] Win32: make FILETIME conversion functions public We will use them in the upcoming "FSCache" patches (to accelerate sequential lstat() calls). Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw-posix.h | 18 ++++++++++++++++++ compat/mingw.c | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/compat/mingw-posix.h b/compat/mingw-posix.h index 9dc70e1c49cdf5..96df96f8e49c14 100644 --- a/compat/mingw-posix.h +++ b/compat/mingw-posix.h @@ -340,6 +340,17 @@ static inline int getrlimit(int resource, struct rlimit *rlp) return 0; } +/* + * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. + * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. + */ +static inline long long filetime_to_hnsec(const FILETIME *ft) +{ + long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; + /* Windows to Unix Epoch conversion */ + return winTime - 116444736000000000LL; +} + /* * Use mingw specific stat()/lstat()/fstat() implementations on Windows, * including our own struct stat with 64 bit st_size and nanosecond-precision @@ -356,6 +367,13 @@ struct timespec { #endif #endif +static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) +{ + long long hnsec = filetime_to_hnsec(ft); + ts->tv_sec = (time_t)(hnsec / 10000000); + ts->tv_nsec = (hnsec % 10000000) * 100; +} + struct mingw_stat { _dev_t st_dev; _ino_t st_ino; diff --git a/compat/mingw.c b/compat/mingw.c index af2806f7519162..adf2d0a16d91ba 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1181,24 +1181,6 @@ int mingw_chmod(const char *filename, int mode) return _wchmod(wfilename, mode); } -/* - * The unit of FILETIME is 100-nanoseconds since January 1, 1601, UTC. - * Returns the 100-nanoseconds ("hekto nanoseconds") since the epoch. - */ -static inline long long filetime_to_hnsec(const FILETIME *ft) -{ - long long winTime = ((long long)ft->dwHighDateTime << 32) + ft->dwLowDateTime; - /* Windows to Unix Epoch conversion */ - return winTime - 116444736000000000LL; -} - -static inline void filetime_to_timespec(const FILETIME *ft, struct timespec *ts) -{ - long long hnsec = filetime_to_hnsec(ft); - ts->tv_sec = (time_t)(hnsec / 10000000); - ts->tv_nsec = (hnsec % 10000000) * 100; -} - /** * Verifies that safe_create_leading_directories() would succeed. */ From 70da5f3b461f6f945cf456d89eeb2015b89fae7c Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:17:31 +0200 Subject: [PATCH 156/229] Win32: dirent.c: Move opendir down Move opendir down in preparation for the next patch. Signed-off-by: Karsten Blees --- compat/win32/dirent.c | 68 +++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 24ee9b814d6adf..2f7d64369f9eda 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -21,40 +21,6 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -DIR *opendir(const char *name) -{ - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; - HANDLE h; - int len; - DIR *dir; - - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) - return NULL; - - /* append optional '/' and wildcard '*' */ - if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; - pattern[len++] = '*'; - pattern[len] = 0; - - /* open find handle */ - h = FindFirstFileW(pattern, &fdata); - if (h == INVALID_HANDLE_VALUE) { - DWORD err = GetLastError(); - errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); - return NULL; - } - - /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); - dir->dd_handle = h; - dir->dd_stat = 0; - finddata2dirent(&dir->dd_dir, &fdata); - return dir; -} - struct dirent *readdir(DIR *dir) { if (!dir) { @@ -93,3 +59,37 @@ int closedir(DIR *dir) free(dir); return 0; } + +DIR *opendir(const char *name) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int len; + DIR *dir; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((len = xutftowcs_path(pattern, name)) < 0) + return NULL; + + /* append optional '/' and wildcard '*' */ + if (len && !is_dir_sep(pattern[len - 1])) + pattern[len++] = '/'; + pattern[len++] = '*'; + pattern[len] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + DWORD err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* initialize DIR structure and copy first dir entry */ + dir = xmalloc(sizeof(DIR)); + dir->dd_handle = h; + dir->dd_stat = 0; + finddata2dirent(&dir->dd_dir, &fdata); + return dir; +} From e4660a366d02e8e221ea9addbb701de4fff8db9d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:18:40 +0200 Subject: [PATCH 157/229] mingw: make the dirent implementation pluggable Emulating the POSIX `dirent` API on Windows via `FindFirstFile()`/`FindNextFile()` is pretty staightforward, however, most of the information provided in the `WIN32_FIND_DATA` structure is thrown away in the process. A more sophisticated implementation may cache this data, e.g. for later reuse in calls to `lstat()`. Make the `dirent` implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Define a base DIR structure with pointers to `readdir()`/`closedir()` that match the `opendir()` implementation (similar to vtable pointers in Object-Oriented Programming). Define `readdir()`/`closedir()` so that they call the function pointers in the `DIR` structure. This allows to choose the `opendir()` implementation on a call-by-call basis. Make the fixed-size `dirent.d_name` buffer a flex array, as `d_name` may be implementation specific (e.g. a caching implementation may allocate a `struct dirent` with _just_ the size needed to hold the `d_name` in question). Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/win32/dirent.c | 30 +++++++++++++++++++----------- compat/win32/dirent.h | 28 +++++++++++++++++++++------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index 2f7d64369f9eda..f17e1595468f44 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -1,15 +1,21 @@ #include "../../git-compat-util.h" -struct DIR { - struct dirent dd_dir; /* includes d_type */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +typedef struct dirent_DIR { + struct DIR base_dir; /* extend base struct DIR */ HANDLE dd_handle; /* FindFirstFile handle */ int dd_stat; /* 0-based index */ -}; + struct dirent dd_dir; /* includes d_type */ +} dirent_DIR; +#pragma GCC diagnostic pop + +DIR *(*opendir)(const char *dirname) = dirent_opendir; static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) { - /* convert UTF-16 name to UTF-8 */ - xwcstoutf(ent->d_name, fdata->cFileName, sizeof(ent->d_name)); + /* convert UTF-16 name to UTF-8 (d_name points to dirent_DIR.dd_name) */ + xwcstoutf(ent->d_name, fdata->cFileName, MAX_PATH * 3); /* Set file type, based on WIN32_FIND_DATA */ if ((fdata->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) @@ -21,7 +27,7 @@ static inline void finddata2dirent(struct dirent *ent, WIN32_FIND_DATAW *fdata) ent->d_type = DT_REG; } -struct dirent *readdir(DIR *dir) +static struct dirent *dirent_readdir(dirent_DIR *dir) { if (!dir) { errno = EBADF; /* No set_errno for mingw */ @@ -48,7 +54,7 @@ struct dirent *readdir(DIR *dir) return &dir->dd_dir; } -int closedir(DIR *dir) +static int dirent_closedir(dirent_DIR *dir) { if (!dir) { errno = EBADF; @@ -60,13 +66,13 @@ int closedir(DIR *dir) return 0; } -DIR *opendir(const char *name) +DIR *dirent_opendir(const char *name) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ WIN32_FIND_DATAW fdata; HANDLE h; int len; - DIR *dir; + dirent_DIR *dir; /* convert name to UTF-16 and check length < MAX_PATH */ if ((len = xutftowcs_path(pattern, name)) < 0) @@ -87,9 +93,11 @@ DIR *opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(DIR)); + dir = xmalloc(sizeof(dirent_DIR) + MAX_PATH); + dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; + dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; dir->dd_handle = h; dir->dd_stat = 0; finddata2dirent(&dir->dd_dir, &fdata); - return dir; + return (DIR*) dir; } diff --git a/compat/win32/dirent.h b/compat/win32/dirent.h index 058207e4bfed62..a58a8075fd70e3 100644 --- a/compat/win32/dirent.h +++ b/compat/win32/dirent.h @@ -1,20 +1,34 @@ #ifndef DIRENT_H #define DIRENT_H -typedef struct DIR DIR; - #define DT_UNKNOWN 0 #define DT_DIR 1 #define DT_REG 2 #define DT_LNK 3 struct dirent { - unsigned char d_type; /* file type to prevent lstat after readdir */ - char d_name[MAX_PATH * 3]; /* file name (* 3 for UTF-8 conversion) */ + unsigned char d_type; /* file type to prevent lstat after readdir */ + char d_name[/* FLEX_ARRAY */]; /* file name */ }; -DIR *opendir(const char *dirname); -struct dirent *readdir(DIR *dir); -int closedir(DIR *dir); +/* + * Base DIR structure, contains pointers to readdir/closedir implementations so + * that opendir may choose a concrete implementation on a call-by-call basis. + */ +typedef struct DIR { + struct dirent *(*preaddir)(struct DIR *dir); + int (*pclosedir)(struct DIR *dir); +} DIR; + +/* default dirent implementation */ +extern DIR *dirent_opendir(const char *dirname); + +#define opendir git_opendir + +/* current dirent implementation */ +extern DIR *(*opendir)(const char *dirname); + +#define readdir(dir) (dir->preaddir(dir)) +#define closedir(dir) (dir->pclosedir(dir)) #endif /* DIRENT_H */ From 28bf8ef6c8ade01a54e9ebf2ac3386c6c1750248 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:21:30 +0200 Subject: [PATCH 158/229] Win32: make the lstat implementation pluggable Emulating the POSIX lstat API on Windows via GetFileAttributes[Ex] is quite slow. Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. A caching implementation may improve performance by bulk-reading entire directories or reusing data obtained via opendir / readdir. Make the lstat implementation pluggable so that it can be switched at runtime, e.g. based on a config option. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/mingw-posix.h | 2 +- compat/mingw.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compat/mingw-posix.h b/compat/mingw-posix.h index 96df96f8e49c14..9158f89d89d239 100644 --- a/compat/mingw-posix.h +++ b/compat/mingw-posix.h @@ -406,7 +406,7 @@ int mingw_fstat(int fd, struct stat *buf); #ifdef lstat #undef lstat #endif -#define lstat mingw_lstat +extern int (*lstat)(const char *file_name, struct stat *buf); int mingw_utime(const char *file_name, const struct utimbuf *times); diff --git a/compat/mingw.c b/compat/mingw.c index adf2d0a16d91ba..a610dcf1c4853e 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1378,6 +1378,8 @@ int mingw_lstat(const char *file_name, struct stat *buf) return -1; } +int (*lstat)(const char *file_name, struct stat *buf) = mingw_lstat; + static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) { BY_HANDLE_FILE_INFORMATION fdata; From 8423e5d8ff043f559dd511f71f618e1af5a3bf41 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 20 Sep 2017 21:52:28 +0200 Subject: [PATCH 159/229] git-gui--askyesno: fix funny text wrapping The text wrapping seems to be aligned to the right side of the Yes button, leaving an awful lot of empty space. Let's try to counter this by using pixel units. Signed-off-by: Johannes Schindelin --- git-gui/git-gui--askyesno.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/git-gui/git-gui--askyesno.sh b/git-gui/git-gui--askyesno.sh index 142d1bc3de229b..837281fe337b6f 100755 --- a/git-gui/git-gui--askyesno.sh +++ b/git-gui/git-gui--askyesno.sh @@ -29,8 +29,8 @@ if {$argc < 1} { } ${NS}::frame .t -${NS}::label .t.m -text $prompt -justify center -width 40 -.t.m configure -wraplength 400 +${NS}::label .t.m -text $prompt -justify center -width 400px +.t.m configure -wraplength 400px pack .t.m -side top -fill x -padx 20 -pady 20 -expand 1 pack .t -side top -fill x -ipadx 20 -ipady 20 -expand 1 From 90a778e1602c5d7f655bf714d81cd1f9a4a8da01 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Sun, 8 Sep 2013 14:23:27 +0200 Subject: [PATCH 160/229] mingw: add infrastructure for read-only file system level caches Add a macro to mark code sections that only read from the file system, along with a config option and documentation. This facilitates implementation of relatively simple file system level caches without the need to synchronize with the file system. Enable read-only sections for 'git status' and preload_index. Signed-off-by: Karsten Blees --- Documentation/config/core.adoc | 6 ++++++ builtin/commit.c | 1 + compat/mingw.c | 6 ++++++ compat/mingw.h | 2 ++ git-compat-util.h | 15 +++++++++++++++ preload-index.c | 3 +++ 6 files changed, 33 insertions(+) diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index aaafe7a7889f66..ca8da41aa50fdf 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -745,6 +745,12 @@ relatively high IO latencies. When enabled, Git will do the index comparison to the filesystem data in parallel, allowing overlapping IO's. Defaults to true. +core.fscache:: + Enable additional caching of file system data for some operations. ++ +Git for Windows uses this to bulk-read and cache lstat data of entire +directories (instead of doing lstat file by file). + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. diff --git a/builtin/commit.c b/builtin/commit.c index 8422ee1b387e0e..0baee46574e543 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1625,6 +1625,7 @@ struct repository *repo UNUSED) PATHSPEC_PREFER_FULL, prefix, argv); + enable_fscache(1); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) progress_flag = REFRESH_PROGRESS; diff --git a/compat/mingw.c b/compat/mingw.c index a610dcf1c4853e..76fa68348fcf6d 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -278,6 +278,7 @@ enum hide_dotfiles_type { static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; +int core_fscache; int mingw_core_config(const char *var, const char *value, const struct config_context *ctx UNUSED, @@ -291,6 +292,11 @@ int mingw_core_config(const char *var, const char *value, return 0; } + if (!strcmp(var, "core.fscache")) { + core_fscache = git_config_bool(var, value); + return 0; + } + if (!strcmp(var, "core.unsetenvvars")) { if (!value) return config_error_nonbool(var); diff --git a/compat/mingw.h b/compat/mingw.h index c9512194b7fff3..bae25483cf7ce0 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -1,5 +1,7 @@ #include "mingw-posix.h" +extern int core_fscache; + struct config_context; int mingw_core_config(const char *var, const char *value, const struct config_context *ctx, void *cb); diff --git a/git-compat-util.h b/git-compat-util.h index b849cb1b40fdb6..0637c399e52fb8 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1083,6 +1083,21 @@ static inline int is_missing_file_error(int errno_) return (errno_ == ENOENT || errno_ == ENOTDIR); } +/* + * Enable/disable a read-only cache for file system data on platforms that + * support it. + * + * Implementing a live-cache is complicated and requires special platform + * support (inotify, ReadDirectoryChangesW...). enable_fscache shall be used + * to mark sections of git code that extensively read from the file system + * without modifying anything. Implementations can use this to cache e.g. stat + * data or even file content without the need to synchronize with the file + * system. + */ +#ifndef enable_fscache +#define enable_fscache(x) /* noop */ +#endif + int cmd_main(int, const char **); /* diff --git a/preload-index.c b/preload-index.c index b222821b448526..61e8f3a1f6ec84 100644 --- a/preload-index.c +++ b/preload-index.c @@ -141,6 +141,7 @@ void preload_index(struct index_state *index, pthread_mutex_init(&pd.mutex, NULL); } + enable_fscache(1); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; int err; @@ -176,6 +177,8 @@ void preload_index(struct index_state *index, trace2_data_intmax("index", NULL, "preload/sum_lstat", t2_sum_lstat); trace2_region_leave("index", "preload", NULL); + + enable_fscache(0); } int repo_read_index_preload(struct repository *repo, From d907f699cc623a038d797f9b9016dcfb7ae89573 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 20 Sep 2017 21:55:45 +0200 Subject: [PATCH 161/229] git-gui--askyesno (mingw): use Git for Windows' icon, if available For additional GUI goodness. Signed-off-by: Johannes Schindelin --- git-gui/git-gui--askyesno.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/git-gui/git-gui--askyesno.sh b/git-gui/git-gui--askyesno.sh index 837281fe337b6f..e431f86a8e16ae 100755 --- a/git-gui/git-gui--askyesno.sh +++ b/git-gui/git-gui--askyesno.sh @@ -59,5 +59,17 @@ if {$::tcl_platform(platform) eq {windows}} { } } +if {$::tcl_platform(platform) eq {windows}} { + set icopath [file dirname [file normalize $argv0]] + if {[file tail $icopath] eq {git-core}} { + set icopath [file dirname $icopath] + } + set icopath [file dirname $icopath] + set icopath [file join $icopath share git git-for-windows.ico] + if {[file exists $icopath]} { + wm iconbitmap . -default $icopath + } +} + wm title . $title tk::PlaceWindow . From 790a92a7d0b025204d1cccf1c1de687455b755a7 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 1 Oct 2013 12:51:54 +0200 Subject: [PATCH 162/229] mingw: add a cache below mingw's lstat and dirent implementations Checking the work tree status is quite slow on Windows, due to slow `lstat()` emulation (git calls `lstat()` once for each file in the index). Windows operating system APIs seem to be much better at scanning the status of entire directories than checking single files. Add an `lstat()` implementation that uses a cache for lstat data. Cache misses read the entire parent directory and add it to the cache. Subsequent `lstat()` calls for the same directory are served directly from the cache. Also implement `opendir()`/`readdir()`/`closedir()` so that they create and use directory listings in the cache. The cache doesn't track file system changes and doesn't plug into any modifying file APIs, so it has to be explicitly enabled for git functions that don't modify the working copy. Note: in an earlier version of this patch, the cache was always active and tracked file system changes via ReadDirectoryChangesW. However, this was much more complex and had negative impact on the performance of modifying git commands such as 'git checkout'. Signed-off-by: Karsten Blees Signed-off-by: Johannes Schindelin --- compat/win32/fscache.c | 487 ++++++++++++++++++++++++++++ compat/win32/fscache.h | 10 + config.mak.uname | 4 +- contrib/buildsystems/CMakeLists.txt | 3 +- git-compat-util.h | 2 + meson.build | 1 + 6 files changed, 504 insertions(+), 3 deletions(-) create mode 100644 compat/win32/fscache.c create mode 100644 compat/win32/fscache.h diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c new file mode 100644 index 00000000000000..4305514a8bf763 --- /dev/null +++ b/compat/win32/fscache.c @@ -0,0 +1,487 @@ +#include "../../git-compat-util.h" +#include "../../hashmap.h" +#include "../win32.h" +#include "fscache.h" +#include "../../dir.h" +#include "../../abspath.h" + +static int initialized; +static volatile long enabled; +static struct hashmap map; +static CRITICAL_SECTION mutex; + +/* + * An entry in the file system cache. Used for both entire directory listings + * and file entries. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +struct fsentry { + struct hashmap_entry ent; + mode_t st_mode; + /* Pointer to the directory listing, or NULL for the listing itself. */ + struct fsentry *list; + /* Pointer to the next file entry of the list. */ + struct fsentry *next; + + union { + /* Reference count of the directory listing. */ + volatile long refcnt; + struct { + /* More stat members (only used for file entries). */ + off64_t st_size; + struct timespec st_atim; + struct timespec st_mtim; + struct timespec st_ctim; + } s; + } u; + + /* Length of name. */ + unsigned short len; + /* + * Name of the entry. For directory listings: relative path of the + * directory, without trailing '/' (empty for cwd()). For file entries: + * name of the file. Typically points to the end of the structure if + * the fsentry is allocated on the heap (see fsentry_alloc), or to a + * local variable if on the stack (see fsentry_init). + */ + struct dirent dirent; +}; +#pragma GCC diagnostic pop + +#pragma GCC diagnostic push +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" +#endif +struct heap_fsentry { + union { + struct fsentry ent; + char dummy[sizeof(struct fsentry) + MAX_PATH]; + } u; +}; +#pragma GCC diagnostic pop + +/* + * Compares the paths of two fsentry structures for equality. + */ +static int fsentry_cmp(void *cmp_data UNUSED, + const struct fsentry *fse1, const struct fsentry *fse2, + void *keydata UNUSED) +{ + int res; + if (fse1 == fse2) + return 0; + + /* compare the list parts first */ + if (fse1->list != fse2->list && + (res = fsentry_cmp(NULL, fse1->list ? fse1->list : fse1, + fse2->list ? fse2->list : fse2, NULL))) + return res; + + /* if list parts are equal, compare len and name */ + if (fse1->len != fse2->len) + return fse1->len - fse2->len; + return fspathncmp(fse1->dirent.d_name, fse2->dirent.d_name, fse1->len); +} + +/* + * Calculates the hash code of an fsentry structure's path. + */ +static unsigned int fsentry_hash(const struct fsentry *fse) +{ + unsigned int hash = fse->list ? fse->list->ent.hash : 0; + return hash ^ memihash(fse->dirent.d_name, fse->len); +} + +/* + * Initialize an fsentry structure for use by fsentry_hash and fsentry_cmp. + */ +static void fsentry_init(struct fsentry *fse, struct fsentry *list, + const char *name, size_t len) +{ + fse->list = list; + if (len > MAX_PATH) + BUG("Trying to allocate fsentry for long path '%.*s'", + (int)len, name); + memcpy(fse->dirent.d_name, name, len); + fse->dirent.d_name[len] = 0; + fse->len = len; + hashmap_entry_init(&fse->ent, fsentry_hash(fse)); +} + +/* + * Allocate an fsentry structure on the heap. + */ +static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, + size_t len) +{ + /* overallocate fsentry and copy the name to the end */ + struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + /* init the rest of the structure */ + fsentry_init(fse, list, name, len); + fse->next = NULL; + fse->u.refcnt = 1; + return fse; +} + +/* + * Add a reference to an fsentry. + */ +inline static void fsentry_addref(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + InterlockedIncrement(&(fse->u.refcnt)); +} + +/* + * Release the reference to an fsentry, frees the memory if its the last ref. + */ +static void fsentry_release(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + if (InterlockedDecrement(&(fse->u.refcnt))) + return; + + while (fse) { + struct fsentry *next = fse->next; + free(fse); + fse = next; + } +} + +/* + * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. + */ +static struct fsentry *fseentry_create_entry(struct fsentry *list, + const WIN32_FIND_DATAW *fdata) +{ + char buf[MAX_PATH * 3]; + int len; + struct fsentry *fse; + len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); + + fse = fsentry_alloc(list, buf, len); + + fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes, + IO_REPARSE_TAG_SYMLINK); + fse->dirent.d_type = S_ISREG(fse->st_mode) ? DT_REG : + S_ISDIR(fse->st_mode) ? DT_DIR : DT_LNK; + fse->u.s.st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) + | fdata->nFileSizeLow; + filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->u.s.st_atim)); + filetime_to_timespec(&(fdata->ftLastWriteTime), &(fse->u.s.st_mtim)); + filetime_to_timespec(&(fdata->ftCreationTime), &(fse->u.s.st_ctim)); + + return fse; +} + +/* + * Create an fsentry-based directory listing (similar to opendir / readdir). + * Dir should not contain trailing '/'. Use an empty string for the current + * directory (not "."!). + */ +static struct fsentry *fsentry_create_list(const struct fsentry *dir) +{ + wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + WIN32_FIND_DATAW fdata; + HANDLE h; + int wlen; + struct fsentry *list, **phead; + DWORD err; + + /* convert name to UTF-16 and check length < MAX_PATH */ + if ((wlen = xutftowcsn(pattern, dir->dirent.d_name, MAX_PATH, + dir->len)) < 0) { + if (errno == ERANGE) + errno = ENAMETOOLONG; + return NULL; + } + + /* append optional '/' and wildcard '*' */ + if (wlen) + pattern[wlen++] = '/'; + pattern[wlen++] = '*'; + pattern[wlen] = 0; + + /* open find handle */ + h = FindFirstFileW(pattern, &fdata); + if (h == INVALID_HANDLE_VALUE) { + err = GetLastError(); + errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + return NULL; + } + + /* allocate object to hold directory listing */ + list = fsentry_alloc(NULL, dir->dirent.d_name, dir->len); + + /* walk directory and build linked list of fsentry structures */ + phead = &list->next; + do { + *phead = fseentry_create_entry(list, &fdata); + phead = &(*phead)->next; + } while (FindNextFileW(h, &fdata)); + + /* remember result of last FindNextFile, then close find handle */ + err = GetLastError(); + FindClose(h); + + /* return the list if we've got all the files */ + if (err == ERROR_NO_MORE_FILES) + return list; + + /* otherwise free the list and return error */ + fsentry_release(list); + errno = err_win_to_posix(err); + return NULL; +} + +/* + * Adds a directory listing to the cache. + */ +static void fscache_add(struct fsentry *fse) +{ + if (fse->list) + fse = fse->list; + + for (; fse; fse = fse->next) + hashmap_add(&map, &fse->ent); +} + +/* + * Clears the cache. + */ +static void fscache_clear(void) +{ + hashmap_clear_and_free(&map, struct fsentry, ent); + hashmap_init(&map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); +} + +/* + * Checks if the cache is enabled for the given path. + */ +static inline int fscache_enabled(const char *path) +{ + return enabled > 0 && !is_absolute_path(path); +} + +/* + * Looks up or creates a cache entry for the specified key. + */ +static struct fsentry *fscache_get(struct fsentry *key) +{ + struct fsentry *fse; + + EnterCriticalSection(&mutex); + /* check if entry is in cache */ + fse = hashmap_get_entry(&map, key, ent, NULL); + if (fse) { + fsentry_addref(fse); + LeaveCriticalSection(&mutex); + return fse; + } + /* if looking for a file, check if directory listing is in cache */ + if (!fse && key->list) { + fse = hashmap_get_entry(&map, key->list, ent, NULL); + if (fse) { + LeaveCriticalSection(&mutex); + /* dir entry without file entry -> file doesn't exist */ + errno = ENOENT; + return NULL; + } + } + + /* create the directory listing (outside mutex!) */ + LeaveCriticalSection(&mutex); + fse = fsentry_create_list(key->list ? key->list : key); + if (!fse) + return NULL; + + EnterCriticalSection(&mutex); + /* add directory listing if it hasn't been added by some other thread */ + if (!hashmap_get_entry(&map, key, ent, NULL)) + fscache_add(fse); + + /* lookup file entry if requested (fse already points to directory) */ + if (key->list) + fse = hashmap_get_entry(&map, key, ent, NULL); + + /* return entry or ENOENT */ + if (fse) + fsentry_addref(fse); + else + errno = ENOENT; + + LeaveCriticalSection(&mutex); + return fse; +} + +/* + * Enables or disables the cache. Note that the cache is read-only, changes to + * the working directory are NOT reflected in the cache while enabled. + */ +int fscache_enable(int enable) +{ + int result; + + if (!initialized) { + /* allow the cache to be disabled entirely */ + if (!core_fscache) + return 0; + + InitializeCriticalSection(&mutex); + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, 0); + initialized = 1; + } + + result = enable ? InterlockedIncrement(&enabled) + : InterlockedDecrement(&enabled); + + if (enable && result == 1) { + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } else if (!enable && !result) { + /* reset opendir and lstat to the original implementations */ + opendir = dirent_opendir; + lstat = mingw_lstat; + EnterCriticalSection(&mutex); + fscache_clear(); + LeaveCriticalSection(&mutex); + } + return result; +} + +/* + * Lstat replacement, uses the cache if enabled, otherwise redirects to + * mingw_lstat. + */ +int fscache_lstat(const char *filename, struct stat *st) +{ + int dirlen, base, len; +#pragma GCC diagnostic push +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" +#endif + struct heap_fsentry key[2]; +#pragma GCC diagnostic pop + struct fsentry *fse; + + if (!fscache_enabled(filename)) + return mingw_lstat(filename, st); + + /* split filename into path + name */ + len = strlen(filename); + if (len && is_dir_sep(filename[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(filename[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(&key[0].u.ent, NULL, filename, dirlen); + fsentry_init(&key[1].u.ent, &key[0].u.ent, filename + base, len - base); + fse = fscache_get(&key[1].u.ent); + if (!fse) { + errno = ENOENT; + return -1; + } + + /* + * Special case symbolic links: FindFirstFile()/FindNextFile() did not + * provide us with the length of the target path. + */ + if (fse->u.s.st_size == MAX_PATH && S_ISLNK(fse->st_mode)) { + char buf[MAX_PATH]; + int len = readlink(filename, buf, sizeof(buf) - 1); + + if (len > 0) + fse->u.s.st_size = len; + } + + /* copy stat data */ + st->st_ino = 0; + st->st_gid = 0; + st->st_uid = 0; + st->st_dev = 0; + st->st_rdev = 0; + st->st_nlink = 1; + st->st_mode = fse->st_mode; + st->st_size = fse->u.s.st_size; + st->st_atim = fse->u.s.st_atim; + st->st_mtim = fse->u.s.st_mtim; + st->st_ctim = fse->u.s.st_ctim; + + /* don't forget to release fsentry */ + fsentry_release(fse); + return 0; +} + +typedef struct fscache_DIR { + struct DIR base_dir; /* extend base struct DIR */ + struct fsentry *pfsentry; + struct dirent *dirent; +} fscache_DIR; + +/* + * Readdir replacement. + */ +static struct dirent *fscache_readdir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + struct fsentry *next = dir->pfsentry->next; + if (!next) + return NULL; + dir->pfsentry = next; + dir->dirent = &next->dirent; + return dir->dirent; +} + +/* + * Closedir replacement. + */ +static int fscache_closedir(DIR *base_dir) +{ + fscache_DIR *dir = (fscache_DIR*) base_dir; + fsentry_release(dir->pfsentry); + free(dir); + return 0; +} + +/* + * Opendir replacement, uses a directory listing from the cache if enabled, + * otherwise calls original dirent implementation. + */ +DIR *fscache_opendir(const char *dirname) +{ + struct heap_fsentry key; + struct fsentry *list; + fscache_DIR *dir; + int len; + + if (!fscache_enabled(dirname)) + return dirent_opendir(dirname); + + /* prepare name (strip trailing '/', replace '.') */ + len = strlen(dirname); + if ((len == 1 && dirname[0] == '.') || + (len && is_dir_sep(dirname[len - 1]))) + len--; + + /* get directory listing from cache */ + fsentry_init(&key.u.ent, NULL, dirname, len); + list = fscache_get(&key.u.ent); + if (!list) + return NULL; + + /* alloc and return DIR structure */ + dir = (fscache_DIR*) xmalloc(sizeof(fscache_DIR)); + dir->base_dir.preaddir = fscache_readdir; + dir->base_dir.pclosedir = fscache_closedir; + dir->pfsentry = list; + return (DIR*) dir; +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h new file mode 100644 index 00000000000000..ed518b422d705e --- /dev/null +++ b/compat/win32/fscache.h @@ -0,0 +1,10 @@ +#ifndef FSCACHE_H +#define FSCACHE_H + +int fscache_enable(int enable); +#define enable_fscache(x) fscache_enable(x) + +DIR *fscache_opendir(const char *dir); +int fscache_lstat(const char *file_name, struct stat *buf); + +#endif diff --git a/config.mak.uname b/config.mak.uname index dfded3e4b8e18e..65ce13c520eaa9 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -538,7 +538,7 @@ endif compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ compat/win32/trace2_win32_process_info.o \ - compat/win32/dirent.o compat/win32/wsl.o + compat/win32/dirent.o compat/win32/fscache.o compat/win32/wsl.o COMPAT_CFLAGS = -D__USE_MINGW_ACCESS -DDETECT_MSYS_TTY \ -DENSURE_MSYSTEM_IS_SET="\"$(MSYSTEM)\"" -DMINGW_PREFIX="\"$(patsubst /%,%,$(MINGW_PREFIX))\"" \ -DNOGDI -DHAVE_STRING_H -Icompat -Icompat/regex -Icompat/win32 -DSTRIP_EXTENSION=\".exe\" @@ -744,7 +744,7 @@ ifeq ($(uname_S),MINGW) compat/win32/flush.o \ compat/win32/path-utils.o \ compat/win32/pthread.o compat/win32/syslog.o \ - compat/win32/dirent.o compat/win32/wsl.o + compat/win32/dirent.o compat/win32/fscache.o compat/win32/wsl.o BASIC_CFLAGS += -DWIN32 EXTLIBS += -lws2_32 GITLIBS += git.res diff --git a/contrib/buildsystems/CMakeLists.txt b/contrib/buildsystems/CMakeLists.txt index d1636dc050022e..8f4a42e3588a80 100644 --- a/contrib/buildsystems/CMakeLists.txt +++ b/contrib/buildsystems/CMakeLists.txt @@ -301,7 +301,8 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows") compat/win32/trace2_win32_process_info.c compat/win32/dirent.c compat/win32/wsl.c - compat/strdup.c) + compat/strdup.c + compat/win32/fscache.c) set(NO_UNIX_SOCKETS 1) elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") diff --git a/git-compat-util.h b/git-compat-util.h index 0637c399e52fb8..c093e5971805a8 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -158,9 +158,11 @@ static inline int is_xplatform_dir_sep(int c) /* pull in Windows compatibility stuff */ #include "compat/win32/path-utils.h" #include "compat/mingw.h" +#include "compat/win32/fscache.h" #elif defined(_MSC_VER) #include "compat/win32/path-utils.h" #include "compat/msvc.h" +#include "compat/win32/fscache.h" #endif /* used on Mac OS X */ diff --git a/meson.build b/meson.build index 9a0e99e90dc9e2..dbb3cd340d3ece 100644 --- a/meson.build +++ b/meson.build @@ -1315,6 +1315,7 @@ elif host_machine.system() == 'windows' 'compat/winansi.c', 'compat/win32/dirent.c', 'compat/win32/flush.c', + 'compat/win32/fscache.c', 'compat/win32/path-utils.c', 'compat/win32/pthread.c', 'compat/win32/syslog.c', From 3c096103098ae2beba3e6961d0b2728115939c3d Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 24 Jun 2014 13:22:35 +0200 Subject: [PATCH 163/229] fscache: load directories only once If multiple threads access a directory that is not yet in the cache, the directory will be loaded by each thread. Only one of the results is added to the cache, all others are leaked. This wastes performance and memory. On cache miss, add a future object to the cache to indicate that the directory is currently being loaded. Subsequent threads register themselves with the future object and wait. When the first thread has loaded the directory, it replaces the future object with the result and notifies waiting threads. Signed-off-by: Karsten Blees --- compat/win32/fscache.c | 65 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 4305514a8bf763..b87e19f287f566 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -27,6 +27,8 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; + /* Handle to wait on the loading thread. */ + HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -268,16 +270,43 @@ static inline int fscache_enabled(const char *path) return enabled > 0 && !is_absolute_path(path); } +/* + * Looks up a cache entry, waits if its being loaded by another thread. + * The mutex must be owned by the calling thread. + */ +static struct fsentry *fscache_get_wait(struct fsentry *key) +{ + struct fsentry *fse = hashmap_get_entry(&map, key, ent, NULL); + + /* return if its a 'real' entry (future entries have refcnt == 0) */ + if (!fse || fse->list || fse->u.refcnt) + return fse; + + /* create an event and link our key to the future entry */ + key->u.hwait = CreateEvent(NULL, TRUE, FALSE, NULL); + key->next = fse->next; + fse->next = key; + + /* wait for the loading thread to signal us */ + LeaveCriticalSection(&mutex); + WaitForSingleObject(key->u.hwait, INFINITE); + CloseHandle(key->u.hwait); + EnterCriticalSection(&mutex); + + /* repeat cache lookup */ + return hashmap_get_entry(&map, key, ent, NULL); +} + /* * Looks up or creates a cache entry for the specified key. */ static struct fsentry *fscache_get(struct fsentry *key) { - struct fsentry *fse; + struct fsentry *fse, *future, *waiter; EnterCriticalSection(&mutex); /* check if entry is in cache */ - fse = hashmap_get_entry(&map, key, ent, NULL); + fse = fscache_get_wait(key); if (fse) { fsentry_addref(fse); LeaveCriticalSection(&mutex); @@ -285,7 +314,7 @@ static struct fsentry *fscache_get(struct fsentry *key) } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = hashmap_get_entry(&map, key->list, ent, NULL); + fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); /* dir entry without file entry -> file doesn't exist */ @@ -294,16 +323,34 @@ static struct fsentry *fscache_get(struct fsentry *key) } } + /* add future entry to indicate that we're loading it */ + future = key->list ? key->list : key; + future->next = NULL; + future->u.refcnt = 0; + hashmap_add(&map, &future->ent); + /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(key->list ? key->list : key); - if (!fse) + fse = fsentry_create_list(future); + EnterCriticalSection(&mutex); + + /* remove future entry and signal waiting threads */ + hashmap_remove(&map, &future->ent, NULL); + waiter = future->next; + while (waiter) { + HANDLE h = waiter->u.hwait; + waiter = waiter->next; + SetEvent(h); + } + + /* leave on error (errno set by fsentry_create_list) */ + if (!fse) { + LeaveCriticalSection(&mutex); return NULL; + } - EnterCriticalSection(&mutex); - /* add directory listing if it hasn't been added by some other thread */ - if (!hashmap_get_entry(&map, key, ent, NULL)) - fscache_add(fse); + /* add directory listing to the cache */ + fscache_add(fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) From 8e8be9d6f183db17b06c2870da04b0c063ee55d1 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 24 Jan 2017 15:12:13 -0500 Subject: [PATCH 164/229] fscache: add key for GIT_TRACE_FSCACHE Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/win32/fscache.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index b87e19f287f566..a4095c17c9417a 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -4,11 +4,13 @@ #include "fscache.h" #include "../../dir.h" #include "../../abspath.h" +#include "../../trace.h" static int initialized; static volatile long enabled; static struct hashmap map; static CRITICAL_SECTION mutex; +static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); /* * An entry in the file system cache. Used for both entire directory listings @@ -214,6 +216,8 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir) if (h == INVALID_HANDLE_VALUE) { err = GetLastError(); errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); + trace_printf_key(&trace_fscache, "fscache: error(%d) '%s'\n", + errno, dir->dirent.d_name); return NULL; } @@ -399,6 +403,7 @@ int fscache_enable(int enable) fscache_clear(); LeaveCriticalSection(&mutex); } + trace_printf_key(&trace_fscache, "fscache: enable(%d)\n", enable); return result; } From 347bb66c8db5512f562912174db44e113dcf9d9d Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 13 Dec 2016 14:05:32 -0500 Subject: [PATCH 165/229] fscache: remember not-found directories Teach FSCACHE to remember "not found" directories. This is a performance optimization. FSCACHE is a performance optimization available for Windows. It intercepts Posix-style lstat() calls into an in-memory directory using FindFirst/FindNext. It improves performance on Windows by catching the first lstat() call in a directory, using FindFirst/ FindNext to read the list of files (and attribute data) for the entire directory into the cache, and short-cut subsequent lstat() calls in the same directory. This gives a major performance boost on Windows. However, it does not remember "not found" directories. When STATUS runs and there are missing directories, the lstat() interception fails to find the parent directory and simply return ENOENT for the file -- it does not remember that the FindFirst on the directory failed. Thus subsequent lstat() calls in the same directory, each re-attempt the FindFirst. This completely defeats any performance gains. This can be seen by doing a sparse-checkout on a large repo and then doing a read-tree to reset the skip-worktree bits and then running status. This change reduced status times for my very large repo by 60%. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- compat/win32/fscache.c | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index a4095c17c9417a..f16a6fb07e360d 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -188,7 +188,8 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, * Dir should not contain trailing '/'. Use an empty string for the current * directory (not "."!). */ -static struct fsentry *fsentry_create_list(const struct fsentry *dir) +static struct fsentry *fsentry_create_list(const struct fsentry *dir, + int *dir_not_found) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ WIN32_FIND_DATAW fdata; @@ -197,6 +198,8 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir) struct fsentry *list, **phead; DWORD err; + *dir_not_found = 0; + /* convert name to UTF-16 and check length < MAX_PATH */ if ((wlen = xutftowcsn(pattern, dir->dirent.d_name, MAX_PATH, dir->len)) < 0) { @@ -215,6 +218,7 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir) h = FindFirstFileW(pattern, &fdata); if (h == INVALID_HANDLE_VALUE) { err = GetLastError(); + *dir_not_found = 1; /* or empty directory */ errno = (err == ERROR_DIRECTORY) ? ENOTDIR : err_win_to_posix(err); trace_printf_key(&trace_fscache, "fscache: error(%d) '%s'\n", errno, dir->dirent.d_name); @@ -223,6 +227,8 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir) /* allocate object to hold directory listing */ list = fsentry_alloc(NULL, dir->dirent.d_name, dir->len); + list->st_mode = S_IFDIR; + list->dirent.d_type = DT_DIR; /* walk directory and build linked list of fsentry structures */ phead = &list->next; @@ -307,12 +313,16 @@ static struct fsentry *fscache_get_wait(struct fsentry *key) static struct fsentry *fscache_get(struct fsentry *key) { struct fsentry *fse, *future, *waiter; + int dir_not_found; EnterCriticalSection(&mutex); /* check if entry is in cache */ fse = fscache_get_wait(key); if (fse) { - fsentry_addref(fse); + if (fse->st_mode) + fsentry_addref(fse); + else + fse = NULL; /* non-existing directory */ LeaveCriticalSection(&mutex); return fse; } @@ -321,7 +331,10 @@ static struct fsentry *fscache_get(struct fsentry *key) fse = fscache_get_wait(key->list); if (fse) { LeaveCriticalSection(&mutex); - /* dir entry without file entry -> file doesn't exist */ + /* + * dir entry without file entry, or dir does not + * exist -> file doesn't exist + */ errno = ENOENT; return NULL; } @@ -335,7 +348,7 @@ static struct fsentry *fscache_get(struct fsentry *key) /* create the directory listing (outside mutex!) */ LeaveCriticalSection(&mutex); - fse = fsentry_create_list(future); + fse = fsentry_create_list(future, &dir_not_found); EnterCriticalSection(&mutex); /* remove future entry and signal waiting threads */ @@ -349,6 +362,18 @@ static struct fsentry *fscache_get(struct fsentry *key) /* leave on error (errno set by fsentry_create_list) */ if (!fse) { + if (dir_not_found && key->list) { + /* + * Record that the directory does not exist (or is + * empty, which for all practical matters is the same + * thing as far as fscache is concerned). + */ + fse = fsentry_alloc(key->list->list, + key->list->dirent.d_name, + key->list->len); + fse->st_mode = 0; + hashmap_add(&map, &fse->ent); + } LeaveCriticalSection(&mutex); return NULL; } @@ -360,6 +385,9 @@ static struct fsentry *fscache_get(struct fsentry *key) if (key->list) fse = hashmap_get_entry(&map, key, ent, NULL); + if (fse && !fse->st_mode) + fse = NULL; /* non-existing directory */ + /* return entry or ENOENT */ if (fse) fsentry_addref(fse); From 47c302b82c36998d3c03b993de345ea1152bdfaf Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 25 Jan 2017 18:39:16 +0100 Subject: [PATCH 166/229] fscache: add a test for the dir-not-found optimization Signed-off-by: Johannes Schindelin --- t/t1090-sparse-checkout-scope.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/t/t1090-sparse-checkout-scope.sh b/t/t1090-sparse-checkout-scope.sh index 3a14218b245d4c..529844e2862c74 100755 --- a/t/t1090-sparse-checkout-scope.sh +++ b/t/t1090-sparse-checkout-scope.sh @@ -106,4 +106,24 @@ test_expect_success 'in partial clone, sparse checkout only fetches needed blobs test_cmp expect actual ' +test_expect_success MINGW 'no unnecessary opendir() with fscache' ' + git clone . fscache-test && + ( + cd fscache-test && + git config core.fscache 1 && + echo "/excluded/*" >.git/info/sparse-checkout && + for f in $(test_seq 10) + do + sha1=$(echo $f | git hash-object -w --stdin) && + git update-index --add \ + --cacheinfo 100644,$sha1,excluded/$f || exit 1 + done && + test_tick && + git commit -m excluded && + GIT_TRACE_FSCACHE=1 git status >out 2>err && + grep excluded err >grep.out && + test_line_count = 1 grep.out + ) +' + test_done From defb4f6084370d7a79cc7283c6b65ff8b6d7d8ab Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Tue, 22 Nov 2016 11:26:38 -0500 Subject: [PATCH 167/229] add: use preload-index and fscache for performance Teach "add" to use preload-index and fscache features to improve performance on very large repositories. During an "add", a call is made to run_diff_files() which calls check_remove() for each index-entry. This calls lstat(). On Windows, the fscache code intercepts the lstat() calls and builds a private cache using the FindFirst/FindNext routines, which are much faster. Somewhat independent of this, is the preload-index code which distributes some of the start-up costs across multiple threads. We need to keep the call to read_cache() before parsing the pathspecs (and hence cannot use the pathspecs to limit any preload) because parse_pathspec() is using the index to determine whether a pathspec is, in fact, in a submodule. If we would not read the index first, parse_pathspec() would not error out on a path that is inside a submodule, and t7400-submodule-basic.sh would fail with not ok 47 - do not add files from a submodule We still want the nice preload performance boost, though, so we simply call read_cache_preload(&pathspecs) after parsing the pathspecs. Signed-off-by: Jeff Hostetler Signed-off-by: Johannes Schindelin --- builtin/add.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/builtin/add.c b/builtin/add.c index f565d4d0bc46d2..9798e968711706 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -575,12 +575,16 @@ int cmd_add(int argc, (!(addremove || take_worktree_changes) ? ADD_CACHE_IGNORE_REMOVAL : 0)); + enable_fscache(1); if (repo_read_index_preload(repo, &pathspec, 0) < 0) die(_("index file corrupt")); die_in_unpopulated_submodule(repo->index, prefix); die_path_inside_submodule(repo->index, &pathspec); + /* We do not really re-read the index but update the up-to-date flags */ + preload_index(repo->index, &pathspec, 0); + if (add_new_files) { int baselen; @@ -695,5 +699,6 @@ int cmd_add(int argc, free(ps_matched); dir_clear(&dir); clear_pathspec(&pathspec); + enable_fscache(0); return exit_status; } From f1c01e8e8c1094528e4b4f5d6946d9772b7246e9 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 1 Nov 2017 15:05:44 -0400 Subject: [PATCH 168/229] dir.c: make add_excludes aware of fscache during status Teach read_directory_recursive() and add_excludes() to be aware of optional fscache and avoid trying to open() and fstat() non-existant ".gitignore" files in every directory in the worktree. The current code in add_excludes() calls open() and then fstat() for a ".gitignore" file in each directory present in the worktree. Change that when fscache is enabled to call lstat() first and if present, call open(). This seems backwards because both lstat needs to do more work than fstat. But when fscache is enabled, fscache will already know if the .gitignore file exists and can completely avoid the IO calls. This works because of the lstat diversion to mingw_lstat when fscache is enabled. This reduced status times on a 350K file enlistment of the Windows repo on a NVMe SSD by 0.25 seconds. Signed-off-by: Jeff Hostetler --- compat/win32/fscache.c | 5 +++++ compat/win32/fscache.h | 3 +++ dir.c | 39 ++++++++++++++++++++++++++++++--------- git-compat-util.h | 4 ++++ 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index f16a6fb07e360d..7d7699d3322471 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -12,6 +12,11 @@ static struct hashmap map; static CRITICAL_SECTION mutex; static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); +int fscache_is_enabled(void) +{ + return enabled; +} + /* * An entry in the file system cache. Used for both entire directory listings * and file entries. diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h index ed518b422d705e..9a21fd5709c5bc 100644 --- a/compat/win32/fscache.h +++ b/compat/win32/fscache.h @@ -4,6 +4,9 @@ int fscache_enable(int enable); #define enable_fscache(x) fscache_enable(x) +int fscache_is_enabled(void); +#define is_fscache_enabled() (fscache_is_enabled()) + DIR *fscache_opendir(const char *dir); int fscache_lstat(const char *file_name, struct stat *buf); diff --git a/dir.c b/dir.c index b45bef44469421..8fe7de5faa2ac3 100644 --- a/dir.c +++ b/dir.c @@ -1172,16 +1172,37 @@ static int add_patterns(const char *fname, const char *base, int baselen, size_t size = 0; char *buf; - if (flags & PATTERN_NOFOLLOW) - fd = open_nofollow(fname, O_RDONLY); - else - fd = open(fname, O_RDONLY); - - if (fd < 0 || fstat(fd, &st) < 0) { - if (fd < 0) - warn_on_fopen_errors(fname); + /* + * Since `clang`'s `-Wunreachable-code` mode is clever, it would figure + * out that on non-Windows platforms, this `lstat()` is unreachable. + * We do want to keep the conditional block for the sake of Windows, + * though, so let's use the `NOT_CONSTANT()` trick to suppress that error. + */ + if (NOT_CONSTANT(is_fscache_enabled())) { + if (lstat(fname, &st) < 0) { + fd = -1; + } else { + fd = open(fname, O_RDONLY); + if (fd < 0) + warn_on_fopen_errors(fname); + } + } else { + if (flags & PATTERN_NOFOLLOW) + fd = open_nofollow(fname, O_RDONLY); else - close(fd); + fd = open(fname, O_RDONLY); + + if (fd < 0 || fstat(fd, &st) < 0) { + if (fd < 0) + warn_on_fopen_errors(fname); + else { + close(fd); + fd = -1; + } + } + } + + if (fd < 0) { if (!istate) return -1; r = read_skip_worktree_file_from_index(istate, fname, diff --git a/git-compat-util.h b/git-compat-util.h index c093e5971805a8..2f3fa81e3dd1bb 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1100,6 +1100,10 @@ static inline int is_missing_file_error(int errno_) #define enable_fscache(x) /* noop */ #endif +#ifndef is_fscache_enabled +#define is_fscache_enabled() (0) +#endif + int cmd_main(int, const char **); /* From 410950ac053c55a11e4eac456270a92428868408 Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 20 Dec 2017 10:43:41 -0500 Subject: [PATCH 169/229] fscache: make fscache_enabled() public Make fscache_enabled() function public rather than static. Remove unneeded fscache_is_enabled() function. Change is_fscache_enabled() macro to call fscache_enabled(). is_fscache_enabled() now takes a pathname so that the answer is more precise and mean "is fscache enabled for this pathname", since fscache only stores repo-relative paths and not absolute paths, we can avoid attempting lookups for absolute paths. Signed-off-by: Jeff Hostetler --- compat/win32/fscache.c | 7 +------ compat/win32/fscache.h | 4 ++-- dir.c | 2 +- git-compat-util.h | 2 +- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 7d7699d3322471..ef0a2686a66fb9 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -12,11 +12,6 @@ static struct hashmap map; static CRITICAL_SECTION mutex; static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); -int fscache_is_enabled(void) -{ - return enabled; -} - /* * An entry in the file system cache. Used for both entire directory listings * and file entries. @@ -280,7 +275,7 @@ static void fscache_clear(void) /* * Checks if the cache is enabled for the given path. */ -static inline int fscache_enabled(const char *path) +int fscache_enabled(const char *path) { return enabled > 0 && !is_absolute_path(path); } diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h index 9a21fd5709c5bc..660ada053b4309 100644 --- a/compat/win32/fscache.h +++ b/compat/win32/fscache.h @@ -4,8 +4,8 @@ int fscache_enable(int enable); #define enable_fscache(x) fscache_enable(x) -int fscache_is_enabled(void); -#define is_fscache_enabled() (fscache_is_enabled()) +int fscache_enabled(const char *path); +#define is_fscache_enabled(path) fscache_enabled(path) DIR *fscache_opendir(const char *dir); int fscache_lstat(const char *file_name, struct stat *buf); diff --git a/dir.c b/dir.c index 8fe7de5faa2ac3..c63cd419380fa7 100644 --- a/dir.c +++ b/dir.c @@ -1178,7 +1178,7 @@ static int add_patterns(const char *fname, const char *base, int baselen, * We do want to keep the conditional block for the sake of Windows, * though, so let's use the `NOT_CONSTANT()` trick to suppress that error. */ - if (NOT_CONSTANT(is_fscache_enabled())) { + if (NOT_CONSTANT(is_fscache_enabled(fname))) { if (lstat(fname, &st) < 0) { fd = -1; } else { diff --git a/git-compat-util.h b/git-compat-util.h index 2f3fa81e3dd1bb..2aa8e8a731e254 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1101,7 +1101,7 @@ static inline int is_missing_file_error(int errno_) #endif #ifndef is_fscache_enabled -#define is_fscache_enabled() (0) +#define is_fscache_enabled(path) (0) #endif int cmd_main(int, const char **); From 03cc34c932e6fc4f6a09e6be22ce3cc0d431ee2f Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Wed, 20 Dec 2017 11:19:27 -0500 Subject: [PATCH 170/229] dir.c: regression fix for add_excludes with fscache Fix regression described in: https://github.com/git-for-windows/git/issues/1392 which was introduced in: https://github.com/git-for-windows/git/commit/b2353379bba414e6c00dde913497cc9c827366f2 Problem Symptoms ================ When the user has a .gitignore file that is a symlink, the fscache optimization introduced above caused the stat-data from the symlink, rather that of the target file, to be returned. Later when the ignore file was read, the buffer length did not match the stat.st_size field and we called die("cannot use as an exclude file") Optimization Rationale ====================== The above optimization calls lstat() before open() primarily to ask fscache if the file exists. It gets the current stat-data as a side effect essentially for free (since we already have it in memory). If the file does not exist, it does not need to call open(). And since very few directories have .gitignore files, we can greatly reduce time spent in the filesystem. Discussion of Fix ================= The above optimization calls lstat() rather than stat() because the fscache only intercepts lstat() calls. Calls to stat() stay directed to the mingw_stat() completly bypassing fscache. Furthermore, calls to mingw_stat() always call {open, fstat, close} so that symlinks are properly dereferenced, which adds *additional* open/close calls on top of what the original code in dir.c is doing. Since the problem only manifests for symlinks, we add code to overwrite the stat-data when the path is a symlink. This preserves the effect of the performance gains provided by the fscache in the normal case. Signed-off-by: Jeff Hostetler --- dir.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/dir.c b/dir.c index c63cd419380fa7..7c787fa4263add 100644 --- a/dir.c +++ b/dir.c @@ -1173,6 +1173,28 @@ static int add_patterns(const char *fname, const char *base, int baselen, char *buf; /* + * A performance optimization for status. + * + * During a status scan, git looks in each directory for a .gitignore + * file before scanning the directory. Since .gitignore files are not + * that common, we can waste a lot of time looking for files that are + * not there. Fortunately, the fscache already knows if the directory + * contains a .gitignore file, since it has already read the directory + * and it already has the stat-data. + * + * If the fscache is enabled, use the fscache-lstat() interlude to see + * if the file exists (in the fscache hash maps) before trying to open() + * it. + * + * This causes problem when the .gitignore file is a symlink, because + * we call lstat() rather than stat() on the symlnk and the resulting + * stat-data is for the symlink itself rather than the target file. + * We CANNOT use stat() here because the fscache DOES NOT install an + * interlude for stat() and mingw_stat() always calls "open-fstat-close" + * on the file and defeats the purpose of the optimization here. Since + * symlinks are even more rare than .gitignore files, we force a fstat() + * after our open() to get stat-data for the target file. + * * Since `clang`'s `-Wunreachable-code` mode is clever, it would figure * out that on non-Windows platforms, this `lstat()` is unreachable. * We do want to keep the conditional block for the sake of Windows, @@ -1185,6 +1207,11 @@ static int add_patterns(const char *fname, const char *base, int baselen, fd = open(fname, O_RDONLY); if (fd < 0) warn_on_fopen_errors(fname); + else if (S_ISLNK(st.st_mode) && fstat(fd, &st) < 0) { + warn_on_fopen_errors(fname); + close(fd); + fd = -1; + } } } else { if (flags & PATTERN_NOFOLLOW) From 2c6b194120e4fba53d1d1f872aa20a4c526bf23c Mon Sep 17 00:00:00 2001 From: Takuto Ikuta Date: Wed, 22 Nov 2017 20:39:38 +0900 Subject: [PATCH 171/229] fetch-pack.c: enable fscache for stats under .git/objects When I do git fetch, git call file stats under .git/objects for each refs. This takes time when there are many refs. By enabling fscache, git takes file stats by directory traversing and that improved the speed of fetch-pack for repository having large number of refs. In my windows workstation, this improves the time of `git fetch` for chromium repository like below. I took stats 3 times. * With this patch TotalSeconds: 9.9825165 TotalSeconds: 9.1862075 TotalSeconds: 10.1956256 Avg: 9.78811653333333 * Without this patch TotalSeconds: 15.8406702 TotalSeconds: 15.6248053 TotalSeconds: 15.2085938 Avg: 15.5580231 Signed-off-by: Takuto Ikuta Signed-off-by: Johannes Schindelin --- fetch-pack.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fetch-pack.c b/fetch-pack.c index 0300cf783494b1..a7e76adbba297c 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -804,6 +804,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, save_commit_buffer = 0; trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL); + enable_fscache(1); for (ref = *refs; ref; ref = ref->next) { struct commit *commit; @@ -828,6 +829,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, if (!cutoff || cutoff < commit->date) cutoff = commit->date; } + enable_fscache(0); trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL); /* From 6ad5c20869f885dc47e6bc489762c0aa64726838 Mon Sep 17 00:00:00 2001 From: Takuto Ikuta Date: Tue, 30 Jan 2018 22:42:58 +0900 Subject: [PATCH 172/229] checkout.c: enable fscache for checkout again This is retry of #1419. I added flush_fscache macro to flush cached stats after disk writing with tests for regression reported in #1438 and #1442. git checkout checks each file path in sorted order, so cache flushing does not make performance worse unless we have large number of modified files in a directory containing many files. Using chromium repository, I tested `git checkout .` performance when I delete 10 files in different directories. With this patch: TotalSeconds: 4.307272 TotalSeconds: 4.4863595 TotalSeconds: 4.2975562 Avg: 4.36372923333333 Without this patch: TotalSeconds: 20.9705431 TotalSeconds: 22.4867685 TotalSeconds: 18.8968292 Avg: 20.7847136 I confirmed this patch passed all tests in t/ with core_fscache=1. Signed-off-by: Takuto Ikuta --- builtin/checkout.c | 2 ++ compat/win32/fscache.c | 12 ++++++++++++ compat/win32/fscache.h | 3 +++ entry.c | 3 +++ git-compat-util.h | 4 ++++ parallel-checkout.c | 1 + t/t7201-co.sh | 36 ++++++++++++++++++++++++++++++++++++ 7 files changed, 61 insertions(+) diff --git a/builtin/checkout.c b/builtin/checkout.c index 10c141163178fd..cf7dae4afc619a 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -473,6 +473,7 @@ static int checkout_worktree(const struct checkout_opts *opts, if (pc_workers > 1) init_parallel_checkout(); + enable_fscache(1); for (pos = 0; pos < the_repository->index->cache_nr; pos++) { struct cache_entry *ce = the_repository->index->cache[pos]; if (ce->ce_flags & CE_MATCHED) { @@ -498,6 +499,7 @@ static int checkout_worktree(const struct checkout_opts *opts, errs |= run_parallel_checkout(&state, pc_workers, pc_threshold, NULL, NULL); mem_pool_discard(&ce_mem_pool, should_validate_cache_entries()); + enable_fscache(0); remove_marked_cache_entries(the_repository->index, 1); remove_scheduled_dirs(); errs |= finish_delayed_checkout(&state, opts->show_progress); diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index ef0a2686a66fb9..a91482d19cb1a4 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -435,6 +435,18 @@ int fscache_enable(int enable) return result; } +/* + * Flush cached stats result when fscache is enabled. + */ +void fscache_flush(void) +{ + if (enabled) { + EnterCriticalSection(&mutex); + fscache_clear(); + LeaveCriticalSection(&mutex); + } +} + /* * Lstat replacement, uses the cache if enabled, otherwise redirects to * mingw_lstat. diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h index 660ada053b4309..2f06f8df97dcd0 100644 --- a/compat/win32/fscache.h +++ b/compat/win32/fscache.h @@ -7,6 +7,9 @@ int fscache_enable(int enable); int fscache_enabled(const char *path); #define is_fscache_enabled(path) fscache_enabled(path) +void fscache_flush(void); +#define flush_fscache() fscache_flush() + DIR *fscache_opendir(const char *dir); int fscache_lstat(const char *file_name, struct stat *buf); diff --git a/entry.c b/entry.c index b9313b6964daea..5fd4b08991c6ed 100644 --- a/entry.c +++ b/entry.c @@ -423,6 +423,9 @@ static int write_entry(struct cache_entry *ce, char *path, struct conv_attrs *ca } finish: + /* Flush cached lstat in fscache after writing to disk. */ + flush_fscache(); + if (state->refresh_cache) { if (!fstat_done && lstat(ce->name, &st) < 0) return error_errno("unable to stat just-written file %s", diff --git a/git-compat-util.h b/git-compat-util.h index 2aa8e8a731e254..e7c350dabdc499 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1104,6 +1104,10 @@ static inline int is_missing_file_error(int errno_) #define is_fscache_enabled(path) (0) #endif +#ifndef flush_fscache +#define flush_fscache() /* noop */ +#endif + int cmd_main(int, const char **); /* diff --git a/parallel-checkout.c b/parallel-checkout.c index a6d07dcb1805b9..1eb277a0fc0a55 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -647,6 +647,7 @@ static void write_items_sequentially(struct checkout *state) { size_t i; + flush_fscache(); for (i = 0; i < parallel_checkout.nr; i++) { struct parallel_checkout_item *pc_item = ¶llel_checkout.items[i]; write_pc_item(pc_item, state); diff --git a/t/t7201-co.sh b/t/t7201-co.sh index 7d0dcf8c8b6d68..0f23f2ba716da0 100755 --- a/t/t7201-co.sh +++ b/t/t7201-co.sh @@ -35,6 +35,42 @@ fill () { } +test_expect_success MINGW 'fscache flush cache' ' + + git init fscache-test && + cd fscache-test && + git config core.fscache 1 && + echo A > test.txt && + git add test.txt && + git commit -m A && + echo B >> test.txt && + git checkout . && + test -z "$(git status -s)" && + echo A > expect.txt && + test_cmp expect.txt test.txt && + cd .. && + rm -rf fscache-test +' + +test_expect_success MINGW 'fscache flush cache dir' ' + + git init fscache-test && + cd fscache-test && + git config core.fscache 1 && + echo A > test.txt && + git add test.txt && + git commit -m A && + rm test.txt && + mkdir test.txt && + touch test.txt/test.txt && + git checkout . && + test -z "$(git status -s)" && + echo A > expect.txt && + test_cmp expect.txt test.txt && + cd .. && + rm -rf fscache-test +' + test_expect_success setup ' fill x y z >same && fill 1 2 3 4 5 6 7 8 >one && From a2eaa06137f328a938f25e87b854caa41008fb56 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Jul 2017 22:45:01 +0200 Subject: [PATCH 173/229] mingw: explicitly specify with which cmd to prefix the cmdline The main idea of this patch is that even if we have to look up the absolute path of the script, if only the basename was specified as argv[0], then we should use that basename on the command line, too, not the absolute path. This patch will also help with the upcoming patch where we automatically substitute "sh ..." by "busybox sh ..." if "sh" is not in the PATH but "busybox" is: we will do that by substituting the actual executable, but still keep prepending "sh" to the command line. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 672171ee8698f0..c380cf89984449 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -2158,8 +2158,8 @@ static int is_msys2_sh(const char *cmd) } static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaenv, - const char *dir, - int prepend_cmd, int fhin, int fhout, int fherr) + const char *dir, const char *prepend_cmd, + int fhin, int fhout, int fherr) { STARTUPINFOEXW si; PROCESS_INFORMATION pi; @@ -2239,9 +2239,9 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen /* concatenate argv, quoting args as we go */ strbuf_init(&args, 0); if (prepend_cmd) { - char *quoted = (char *)quote_arg(cmd); + char *quoted = (char *)quote_arg(prepend_cmd); strbuf_addstr(&args, quoted); - if (quoted != cmd) + if (quoted != prepend_cmd) free(quoted); } for (; *argv; argv++) { @@ -2361,7 +2361,8 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen return (pid_t)pi.dwProcessId; } -static pid_t mingw_spawnv(const char *cmd, const char **argv, int prepend_cmd) +static pid_t mingw_spawnv(const char *cmd, const char **argv, + const char *prepend_cmd) { return mingw_spawnve_fd(cmd, argv, NULL, NULL, prepend_cmd, 0, 1, 2); } @@ -2389,14 +2390,14 @@ pid_t mingw_spawnvpe(const char *cmd, const char **argv, char **deltaenv, pid = -1; } else { - pid = mingw_spawnve_fd(iprog, argv, deltaenv, dir, 1, + pid = mingw_spawnve_fd(iprog, argv, deltaenv, dir, interpr, fhin, fhout, fherr); free(iprog); } argv[0] = argv0; } else - pid = mingw_spawnve_fd(prog, argv, deltaenv, dir, 0, + pid = mingw_spawnve_fd(prog, argv, deltaenv, dir, NULL, fhin, fhout, fherr); free(prog); } @@ -2421,7 +2422,7 @@ static int try_shell_exec(const char *cmd, char *const *argv) argv2[0] = (char *)cmd; /* full path to the script file */ COPY_ARRAY(&argv2[1], &argv[1], argc); exec_id = trace2_exec(prog, (const char **)argv2); - pid = mingw_spawnv(prog, (const char **)argv2, 1); + pid = mingw_spawnv(prog, (const char **)argv2, interpr); if (pid >= 0) { int status; if (waitpid(pid, &status, 0) < 0) @@ -2445,7 +2446,7 @@ int mingw_execv(const char *cmd, char *const *argv) int exec_id; exec_id = trace2_exec(cmd, (const char **)argv); - pid = mingw_spawnv(cmd, (const char **)argv, 0); + pid = mingw_spawnv(cmd, (const char **)argv, NULL); if (pid < 0) { trace2_exec_result(exec_id, -1); return -1; From c0f06a41f0dbea0f8374dffc597775796210eecd Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Fri, 7 Sep 2018 11:39:57 -0400 Subject: [PATCH 174/229] Enable the filesystem cache (fscache) in refresh_index(). On file systems that support it, this can dramatically speed up operations like add, commit, describe, rebase, reset, rm that would otherwise have to lstat() every file to "re-match" the stat information in the index to that of the file system. On a synthetic repo with 1M files, "git reset" dropped from 52.02 seconds to 14.42 seconds for a savings of 72%. Signed-off-by: Ben Peart --- read-cache.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/read-cache.c b/read-cache.c index 222148f387c95b..20ea28be826c9d 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1527,6 +1527,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, typechange_fmt = in_porcelain ? "T\t%s\n" : "%s: needs update\n"; added_fmt = in_porcelain ? "A\t%s\n" : "%s: needs update\n"; unmerged_fmt = in_porcelain ? "U\t%s\n" : "%s: needs merge\n"; + enable_fscache(1); /* * Use the multi-threaded preload_index() to refresh most of the * cache entries quickly then in the single threaded loop below, @@ -1621,6 +1622,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, display_progress(progress, istate->cache_nr); stop_progress(&progress); trace_performance_leave("refresh index"); + enable_fscache(0); return has_errors; } From 9a6788b66c7aa1f2639c37ce1ac21d58b9e449b0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Jul 2017 20:41:29 +0200 Subject: [PATCH 175/229] mingw: when path_lookup() failed, try BusyBox BusyBox comes with a ton of applets ("applet" being the identical concept to Git's "builtins"). And similar to Git's builtins, the applets can be called via `busybox `, or the BusyBox executable can be copied/hard-linked to the command name. The similarities do not end here. Just as with Git's builtins, it is problematic that BusyBox' hard-linked applets cannot easily be put into a .zip file: .zip archives have no concept of hard-links and therefore would store identical copies (and also extract identical copies, "inflating" the archive unnecessarily). To counteract that issue, MinGit already ships without hard-linked copies of the builtins, and the plan is to do the same with BusyBox' applets: simply ship busybox.exe as single executable, without hard-linked applets. To accommodate that, Git is being taught by this commit a very special trick, exploiting the fact that it is possible to call an executable with a command-line whose argv[0] is different from the executable's name: when `sh` is to be spawned, and no `sh` is found in the PATH, but busybox.exe is, use that executable (with unchanged argv). Likewise, if any executable to be spawned is not on the PATH, but busybox.exe is found, parse the output of `busybox.exe --help` to find out what applets are included, and if the command matches an included applet name, use busybox.exe to execute it. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 63 ++++++++++++++++++++++++++++++++++++++++++++++++ t/t0014-alias.sh | 2 +- 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index c380cf89984449..fd014eef498e55 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -13,6 +13,7 @@ #include "repository.h" #include "run-command.h" #include "strbuf.h" +#include "string-list.h" #include "symlinks.h" #include "trace2.h" #include "win32.h" @@ -1927,6 +1928,65 @@ static char *lookup_prog(const char *dir, int dirlen, const char *cmd, return NULL; } +static char *path_lookup(const char *cmd, int exe_only); + +static char *is_busybox_applet(const char *cmd) +{ + static struct string_list applets = STRING_LIST_INIT_DUP; + static char *busybox_path; + static int busybox_path_initialized; + + /* Avoid infinite loop */ + if (!strncasecmp(cmd, "busybox", 7) && + (!cmd[7] || !strcasecmp(cmd + 7, ".exe"))) + return NULL; + + if (!busybox_path_initialized) { + busybox_path = path_lookup("busybox.exe", 1); + busybox_path_initialized = 1; + } + + /* Assume that sh is compiled in... */ + if (!busybox_path || !strcasecmp(cmd, "sh")) + return xstrdup_or_null(busybox_path); + + if (!applets.nr) { + struct child_process cp = CHILD_PROCESS_INIT; + struct strbuf buf = STRBUF_INIT; + char *p; + + strvec_pushl(&cp.args, busybox_path, "--help", NULL); + + if (capture_command(&cp, &buf, 2048)) { + string_list_append(&applets, ""); + return NULL; + } + + /* parse output */ + p = strstr(buf.buf, "Currently defined functions:\n"); + if (!p) { + warning("Could not parse output of busybox --help"); + string_list_append(&applets, ""); + return NULL; + } + p = strchrnul(p, '\n'); + for (;;) { + size_t len; + + p += strspn(p, "\n\t ,"); + len = strcspn(p, "\n\t ,"); + if (!len) + break; + p[len] = '\0'; + string_list_insert(&applets, p); + p = p + len + 1; + } + } + + return string_list_has_string(&applets, cmd) ? + xstrdup(busybox_path) : NULL; +} + /* * Determines the absolute path of cmd using the split path in path. * If cmd contains a slash or backslash, no lookup is performed. @@ -1955,6 +2015,9 @@ static char *path_lookup(const char *cmd, int exe_only) path = sep + 1; } + if (!prog && !isexe) + prog = is_busybox_applet(cmd); + return prog; } diff --git a/t/t0014-alias.sh b/t/t0014-alias.sh index d2501fc8d643b7..891c874bad1ba4 100755 --- a/t/t0014-alias.sh +++ b/t/t0014-alias.sh @@ -61,7 +61,7 @@ test_expect_success HAVE_DEPRECATED 'looping aliases - deprecated builtins' ' test_expect_success 'run-command formats empty args properly' ' test_must_fail env GIT_TRACE=1 git frotz a "" b " " c 2>actual.raw && - sed -ne "/run_command:/s/.*trace: run_command: //p" actual.raw >actual && + sed -ne "/run_command: git-frotz/s/.*trace: run_command: //p" actual.raw >actual && echo "git-frotz a '\'''\'' b '\'' '\'' c" >expect && test_cmp expect actual ' From b7e19ca925c2a53de4d2e19d3fe5139312aab80f Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 4 Oct 2018 18:10:21 -0400 Subject: [PATCH 176/229] mem_pool: add GIT_TRACE_MEMPOOL support Add tracing around initializing and discarding mempools. In discard report on the amount of memory unused in the current block to help tune setting the initial_size. Signed-off-by: Ben Peart --- mem-pool.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/mem-pool.c b/mem-pool.c index 8bc77cb0e80a35..89bca70f713692 100644 --- a/mem-pool.c +++ b/mem-pool.c @@ -7,7 +7,9 @@ #include "git-compat-util.h" #include "mem-pool.h" #include "gettext.h" +#include "trace.h" +static struct trace_key trace_mem_pool = TRACE_KEY_INIT(MEMPOOL); #define BLOCK_GROWTH_SIZE (1024 * 1024 - sizeof(struct mp_block)) /* @@ -65,12 +67,20 @@ void mem_pool_init(struct mem_pool *pool, size_t initial_size) if (initial_size > 0) mem_pool_alloc_block(pool, initial_size, NULL); + + trace_printf_key(&trace_mem_pool, + "mem_pool (%p): init (%"PRIuMAX") initial size\n", + (void *)pool, (uintmax_t)initial_size); } void mem_pool_discard(struct mem_pool *pool, int invalidate_memory) { struct mp_block *block, *block_to_free; + trace_printf_key(&trace_mem_pool, + "mem_pool (%p): discard (%"PRIuMAX") unused\n", + (void *)pool, + (uintmax_t)(pool->mp_block->end - pool->mp_block->next_free)); block = pool->mp_block; while (block) { From 8ce0b273e8828f4312b159b120c31c9ebae98145 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 23 Oct 2018 11:42:06 -0400 Subject: [PATCH 177/229] fscache: use FindFirstFileExW to avoid retrieving the short name Use FindFirstFileExW with FindExInfoBasic to avoid forcing NTFS to look up the short name. Also switch to a larger (64K vs 4K) buffer using FIND_FIRST_EX_LARGE_FETCH to minimize round trips to the kernel. In a repo with ~200K files, this drops warm cache status times from 3.19 seconds to 2.67 seconds for a 16% savings. Signed-off-by: Ben Peart --- compat/win32/fscache.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index a91482d19cb1a4..a414794ea6a275 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -215,7 +215,8 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir, pattern[wlen] = 0; /* open find handle */ - h = FindFirstFileW(pattern, &fdata); + h = FindFirstFileExW(pattern, FindExInfoBasic, &fdata, FindExSearchNameMatch, + NULL, FIND_FIRST_EX_LARGE_FETCH); if (h == INVALID_HANDLE_VALUE) { err = GetLastError(); *dir_not_found = 1; /* or empty directory */ From d8f75ef85b2ebd6e0434703fa24e1acd19d96844 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Jul 2017 22:18:56 +0200 Subject: [PATCH 178/229] test-tool: learn to act as a drop-in replacement for `iconv` It is convenient to assume that everybody who wants to build & test Git has access to a working `iconv` executable (after all, we already pretty much require libiconv). However, that limits esoteric test scenarios such as Git for Windows', where an end user installation has to ship with `iconv` for the sole purpose of being testable. That payload serves no other purpose. So let's just have a test helper (to be able to test Git, the test helpers have to be available, after all) to act as `iconv` replacement. Signed-off-by: Johannes Schindelin --- Makefile | 1 + t/helper/meson.build | 1 + t/helper/test-iconv.c | 47 +++++++++++++++++++++++++++++++++++++++++++ t/helper/test-tool.c | 1 + t/helper/test-tool.h | 1 + 5 files changed, 51 insertions(+) create mode 100644 t/helper/test-iconv.c diff --git a/Makefile b/Makefile index f5e5d6a04de958..50eef7237f792c 100644 --- a/Makefile +++ b/Makefile @@ -846,6 +846,7 @@ TEST_BUILTINS_OBJS += test-hash-speed.o TEST_BUILTINS_OBJS += test-hash.o TEST_BUILTINS_OBJS += test-hashmap.o TEST_BUILTINS_OBJS += test-hexdump.o +TEST_BUILTINS_OBJS += test-iconv.o TEST_BUILTINS_OBJS += test-json-writer.o TEST_BUILTINS_OBJS += test-lazy-init-name-hash.o TEST_BUILTINS_OBJS += test-match-trees.o diff --git a/t/helper/meson.build b/t/helper/meson.build index 6dbf8c14258ac9..58192240cf67c7 100644 --- a/t/helper/meson.build +++ b/t/helper/meson.build @@ -30,6 +30,7 @@ test_tool_sources = [ 'test-hash.c', 'test-hashmap.c', 'test-hexdump.c', + 'test-iconv.c', 'test-json-writer.c', 'test-lazy-init-name-hash.c', 'test-match-trees.c', diff --git a/t/helper/test-iconv.c b/t/helper/test-iconv.c new file mode 100644 index 00000000000000..d3c772fddf990b --- /dev/null +++ b/t/helper/test-iconv.c @@ -0,0 +1,47 @@ +#include "test-tool.h" +#include "git-compat-util.h" +#include "strbuf.h" +#include "gettext.h" +#include "parse-options.h" +#include "utf8.h" + +int cmd__iconv(int argc, const char **argv) +{ + struct strbuf buf = STRBUF_INIT; + char *from = NULL, *to = NULL, *p; + size_t len; + int ret = 0; + const char * const iconv_usage[] = { + N_("test-helper --iconv []"), + NULL + }; + struct option options[] = { + OPT_STRING('f', "from-code", &from, "encoding", "from"), + OPT_STRING('t', "to-code", &to, "encoding", "to"), + OPT_END() + }; + + argc = parse_options(argc, argv, NULL, options, + iconv_usage, 0); + + if (argc > 1 || !from || !to) + usage_with_options(iconv_usage, options); + + if (!argc) { + if (strbuf_read(&buf, 0, 2048) < 0) + die_errno("Could not read from stdin"); + } else if (strbuf_read_file(&buf, argv[0], 2048) < 0) + die_errno("Could not read from '%s'", argv[0]); + + p = reencode_string_len(buf.buf, buf.len, to, from, &len); + if (!p) + die_errno("Could not reencode"); + if (write(1, p, len) < 0) + ret = !!error_errno("Could not write %"PRIuMAX" bytes", + (uintmax_t)len); + + strbuf_release(&buf); + free(p); + + return ret; +} diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c index e6bf62cf759608..2a72bdfb1d7f63 100644 --- a/t/helper/test-tool.c +++ b/t/helper/test-tool.c @@ -40,6 +40,7 @@ static struct test_cmd cmds[] = { { "hashmap", cmd__hashmap }, { "hash-speed", cmd__hash_speed }, { "hexdump", cmd__hexdump }, + { "iconv", cmd__iconv }, { "json-writer", cmd__json_writer }, { "lazy-init-name-hash", cmd__lazy_init_name_hash }, { "match-trees", cmd__match_trees }, diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h index 9833ac023bdb46..d3cf0570d639c7 100644 --- a/t/helper/test-tool.h +++ b/t/helper/test-tool.h @@ -33,6 +33,7 @@ int cmd__getcwd(int argc, const char **argv); int cmd__hashmap(int argc, const char **argv); int cmd__hash_speed(int argc, const char **argv); int cmd__hexdump(int argc, const char **argv); +int cmd__iconv(int argc, const char **argv); int cmd__json_writer(int argc, const char **argv); int cmd__lazy_init_name_hash(int argc, const char **argv); int cmd__match_trees(int argc, const char **argv); From cd97acc36308f95cab0d48c2e56d430a6e864844 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Fri, 2 Nov 2018 11:19:10 -0400 Subject: [PATCH 179/229] fscache: fscache takes an initial size Update enable_fscache() to take an optional initial size parameter which is used to initialize the hashmap so that it can avoid having to rehash as additional entries are added. Add a separate disable_fscache() macro to make the code clearer and easier to read. Signed-off-by: Ben Peart Signed-off-by: Johannes Schindelin --- builtin/add.c | 4 ++-- builtin/checkout.c | 4 ++-- builtin/commit.c | 4 ++-- compat/win32/fscache.c | 8 ++++++-- compat/win32/fscache.h | 5 +++-- fetch-pack.c | 4 ++-- git-compat-util.h | 4 ++++ preload-index.c | 4 ++-- read-cache.c | 4 ++-- unpack-trees.c | 2 +- 10 files changed, 26 insertions(+), 17 deletions(-) diff --git a/builtin/add.c b/builtin/add.c index 9798e968711706..31cf7daaa4c155 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -575,7 +575,7 @@ int cmd_add(int argc, (!(addremove || take_worktree_changes) ? ADD_CACHE_IGNORE_REMOVAL : 0)); - enable_fscache(1); + enable_fscache(0); if (repo_read_index_preload(repo, &pathspec, 0) < 0) die(_("index file corrupt")); @@ -699,6 +699,6 @@ int cmd_add(int argc, free(ps_matched); dir_clear(&dir); clear_pathspec(&pathspec); - enable_fscache(0); + disable_fscache(); return exit_status; } diff --git a/builtin/checkout.c b/builtin/checkout.c index cf7dae4afc619a..49fa848066ad58 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -473,7 +473,7 @@ static int checkout_worktree(const struct checkout_opts *opts, if (pc_workers > 1) init_parallel_checkout(); - enable_fscache(1); + enable_fscache(the_repository->index->cache_nr); for (pos = 0; pos < the_repository->index->cache_nr; pos++) { struct cache_entry *ce = the_repository->index->cache[pos]; if (ce->ce_flags & CE_MATCHED) { @@ -499,7 +499,7 @@ static int checkout_worktree(const struct checkout_opts *opts, errs |= run_parallel_checkout(&state, pc_workers, pc_threshold, NULL, NULL); mem_pool_discard(&ce_mem_pool, should_validate_cache_entries()); - enable_fscache(0); + disable_fscache(); remove_marked_cache_entries(the_repository->index, 1); remove_scheduled_dirs(); errs |= finish_delayed_checkout(&state, opts->show_progress); diff --git a/builtin/commit.c b/builtin/commit.c index 2eccf5d2f11fff..8a4cf3ab0d8061 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1625,7 +1625,7 @@ struct repository *repo UNUSED) PATHSPEC_PREFER_FULL, prefix, argv); - enable_fscache(1); + enable_fscache(0); if (status_format != STATUS_FORMAT_PORCELAIN && status_format != STATUS_FORMAT_PORCELAIN_V2) progress_flag = REFRESH_PROGRESS; @@ -1666,7 +1666,7 @@ struct repository *repo UNUSED) wt_status_print(&s); wt_status_collect_free_buffers(&s); - enable_fscache(0); + disable_fscache(); return 0; } diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index ad9fdd36964d93..db37cc930a7ab6 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -412,7 +412,7 @@ static struct fsentry *fscache_get(struct fsentry *key) * Enables or disables the cache. Note that the cache is read-only, changes to * the working directory are NOT reflected in the cache while enabled. */ -int fscache_enable(int enable) +int fscache_enable(int enable, size_t initial_size) { int result; @@ -428,7 +428,11 @@ int fscache_enable(int enable) InitializeCriticalSection(&mutex); lstat_requests = opendir_requests = 0; fscache_misses = fscache_requests = 0; - hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, 0); + /* + * avoid having to rehash by leaving room for the parent dirs. + * '4' was determined empirically by testing several repos + */ + hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, initial_size * 4); initialized = 1; } diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h index 2f06f8df97dcd0..d49c9381114da6 100644 --- a/compat/win32/fscache.h +++ b/compat/win32/fscache.h @@ -1,8 +1,9 @@ #ifndef FSCACHE_H #define FSCACHE_H -int fscache_enable(int enable); -#define enable_fscache(x) fscache_enable(x) +int fscache_enable(int enable, size_t initial_size); +#define enable_fscache(initial_size) fscache_enable(1, initial_size) +#define disable_fscache() fscache_enable(0, 0) int fscache_enabled(const char *path); #define is_fscache_enabled(path) fscache_enabled(path) diff --git a/fetch-pack.c b/fetch-pack.c index a7e76adbba297c..1d61a0af937675 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -804,7 +804,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, save_commit_buffer = 0; trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL); - enable_fscache(1); + enable_fscache(0); for (ref = *refs; ref; ref = ref->next) { struct commit *commit; @@ -829,7 +829,7 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator, if (!cutoff || cutoff < commit->date) cutoff = commit->date; } - enable_fscache(0); + disable_fscache(); trace2_region_leave("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL); /* diff --git a/git-compat-util.h b/git-compat-util.h index e7c350dabdc499..0a1dc60210a4ae 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1100,6 +1100,10 @@ static inline int is_missing_file_error(int errno_) #define enable_fscache(x) /* noop */ #endif +#ifndef disable_fscache +#define disable_fscache() /* noop */ +#endif + #ifndef is_fscache_enabled #define is_fscache_enabled(path) (0) #endif diff --git a/preload-index.c b/preload-index.c index 61e8f3a1f6ec84..e466fef15bcd79 100644 --- a/preload-index.c +++ b/preload-index.c @@ -141,7 +141,7 @@ void preload_index(struct index_state *index, pthread_mutex_init(&pd.mutex, NULL); } - enable_fscache(1); + enable_fscache(index->cache_nr); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; int err; @@ -178,7 +178,7 @@ void preload_index(struct index_state *index, trace2_data_intmax("index", NULL, "preload/sum_lstat", t2_sum_lstat); trace2_region_leave("index", "preload", NULL); - enable_fscache(0); + disable_fscache(); } int repo_read_index_preload(struct repository *repo, diff --git a/read-cache.c b/read-cache.c index 20ea28be826c9d..46b7342774da33 100644 --- a/read-cache.c +++ b/read-cache.c @@ -1527,7 +1527,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, typechange_fmt = in_porcelain ? "T\t%s\n" : "%s: needs update\n"; added_fmt = in_porcelain ? "A\t%s\n" : "%s: needs update\n"; unmerged_fmt = in_porcelain ? "U\t%s\n" : "%s: needs merge\n"; - enable_fscache(1); + enable_fscache(0); /* * Use the multi-threaded preload_index() to refresh most of the * cache entries quickly then in the single threaded loop below, @@ -1622,7 +1622,7 @@ int refresh_index(struct index_state *istate, unsigned int flags, display_progress(progress, istate->cache_nr); stop_progress(&progress); trace_performance_leave("refresh index"); - enable_fscache(0); + disable_fscache(); return has_errors; } diff --git a/unpack-trees.c b/unpack-trees.c index 949b2a4869ea99..7958e1f25848cc 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1828,7 +1828,7 @@ static void mark_new_skip_worktree(struct pattern_list *pl, */ enable_fscache(istate->cache_nr); clear_ce_flags(istate, select_flag, skip_wt_flag, pl, show_progress); - enable_fscache(0); + disable_fscache(); } static void populate_from_existing_patterns(struct unpack_trees_options *o, From 592ee324c7bdd2a353c0abe5e82c3b12e08b6491 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 4 Oct 2018 18:10:21 -0400 Subject: [PATCH 180/229] fscache: add GIT_TEST_FSCACHE support Add support to fscache to enable running the entire test suite with the fscache enabled. Signed-off-by: Ben Peart --- compat/win32/fscache.c | 5 +++++ t/README | 3 +++ 2 files changed, 8 insertions(+) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index a414794ea6a275..935e36e5955c98 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -5,6 +5,7 @@ #include "../../dir.h" #include "../../abspath.h" #include "../../trace.h" +#include "config.h" static int initialized; static volatile long enabled; @@ -408,7 +409,11 @@ int fscache_enable(int enable) int result; if (!initialized) { + int fscache = git_env_bool("GIT_TEST_FSCACHE", -1); + /* allow the cache to be disabled entirely */ + if (fscache != -1) + core_fscache = fscache; if (!core_fscache) return 0; diff --git a/t/README b/t/README index fd7e8504b511e2..dd25129fbba75c 100644 --- a/t/README +++ b/t/README @@ -472,6 +472,9 @@ GIT_TEST_NAME_HASH_VERSION=, when set, causes 'git pack-objects' to assume '--name-hash-version='. +GIT_TEST_FSCACHE= exercises the uncommon fscache code path +which adds a cache below mingw's lstat and dirent implementations. + Naming Tests ------------ From c1e213df8d9b59c81d4ba518216552b90c01c3b5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 20 Jul 2017 22:25:21 +0200 Subject: [PATCH 181/229] tests(mingw): if `iconv` is unavailable, use `test-helper --iconv` Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/t/test-lib.sh b/t/test-lib.sh index a893963f86c155..11dc2be5070ebb 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1732,6 +1732,12 @@ Darwin) test_set_prereq GREP_STRIPS_CR test_set_prereq WINDOWS GIT_TEST_CMP="GIT_DIR=/dev/null git diff --no-index --ignore-cr-at-eol --" + if ! type iconv >/dev/null 2>&1 + then + iconv () { + test-tool iconv "$@" + } + fi ;; *CYGWIN*) test_set_prereq POSIXPERM From 3a4295766e3dd07879c516c7f3c6b98ec74dd5fc Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 11 Dec 2018 12:59:29 +0100 Subject: [PATCH 182/229] fscache: remember the reparse tag for each entry We will use this in the next commit to implement an FSCache-aware version of is_mount_point(). Signed-off-by: Johannes Schindelin --- compat/win32/fscache.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 973ae7efb246c9..46dca7a5635faf 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -46,6 +46,7 @@ static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); struct fsentry { struct hashmap_entry ent; mode_t st_mode; + ULONG reparse_tag; /* Pointer to the directory listing, or NULL for the listing itself. */ struct fsentry *list; /* Pointer to the next file entry of the list. */ @@ -202,6 +203,10 @@ static struct fsentry *fseentry_create_entry(struct fscache *cache, fse = fsentry_alloc(cache, list, buf, len); + fse->reparse_tag = + fdata->FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT ? + fdata->EaSize : 0; + fse->st_mode = file_attr_to_st_mode(fdata->FileAttributes, fdata->EaSize); fse->dirent.d_type = S_ISREG(fse->st_mode) ? DT_REG : From 805f201cb223a0087f7b52ad6993f702566071f5 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 4 Oct 2018 15:38:08 -0400 Subject: [PATCH 183/229] fscache: update fscache to be thread specific instead of global The threading model for fscache has been to have a single, global cache. This puts requirements on it to be thread safe so that callers like preload-index can call it from multiple threads. This was implemented with a single mutex and completion events which introduces contention between the calling threads. Simplify the threading model by making fscache thread specific. This allows us to remove the global mutex and synchronization events entirely and instead associate a fscache with every thread that requests one. This works well with the current multi-threading which divides the cache entries into blocks with a separate thread processing each block. At the end of each worker thread, if there is a fscache on the primary thread, merge the cached results from the worker into the primary thread cache. This enables us to reuse the cache later especially when scanning for untracked files. In testing, this reduced the time spent in preload_index() by about 25% and also reduced the CPU utilization significantly. On a repo with ~200K files, it reduced overall status times by ~12%. Signed-off-by: Ben Peart --- compat/win32/fscache.c | 294 +++++++++++++++++++++++++---------------- compat/win32/fscache.h | 22 ++- git-compat-util.h | 12 ++ preload-index.c | 8 +- 4 files changed, 215 insertions(+), 121 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index db37cc930a7ab6..cd57901aaf67b4 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -7,14 +7,24 @@ #include "../../trace.h" #include "config.h" -static int initialized; -static volatile long enabled; -static struct hashmap map; +static volatile long initialized; +static DWORD dwTlsIndex; static CRITICAL_SECTION mutex; -static unsigned int lstat_requests; -static unsigned int opendir_requests; -static unsigned int fscache_requests; -static unsigned int fscache_misses; + +/* + * Store one fscache per thread to avoid thread contention and locking. + * This is ok because multi-threaded access is 1) uncommon and 2) always + * splitting up the cache entries across multiple threads so there isn't + * any overlap between threads anyway. + */ +struct fscache { + volatile long enabled; + struct hashmap map; + unsigned int lstat_requests; + unsigned int opendir_requests; + unsigned int fscache_requests; + unsigned int fscache_misses; +}; static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); /* @@ -34,8 +44,6 @@ struct fsentry { union { /* Reference count of the directory listing. */ volatile long refcnt; - /* Handle to wait on the loading thread. */ - HANDLE hwait; struct { /* More stat members (only used for file entries). */ off64_t st_size; @@ -260,86 +268,63 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir, /* * Adds a directory listing to the cache. */ -static void fscache_add(struct fsentry *fse) +static void fscache_add(struct fscache *cache, struct fsentry *fse) { if (fse->list) fse = fse->list; for (; fse; fse = fse->next) - hashmap_add(&map, &fse->ent); + hashmap_add(&cache->map, &fse->ent); } /* * Clears the cache. */ -static void fscache_clear(void) +static void fscache_clear(struct fscache *cache) { - hashmap_clear_and_free(&map, struct fsentry, ent); - hashmap_init(&map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); - lstat_requests = opendir_requests = 0; - fscache_misses = fscache_requests = 0; + hashmap_clear_and_free(&cache->map, struct fsentry, ent); + hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); + cache->lstat_requests = cache->opendir_requests = 0; + cache->fscache_misses = cache->fscache_requests = 0; } /* * Checks if the cache is enabled for the given path. */ -int fscache_enabled(const char *path) +static int do_fscache_enabled(struct fscache *cache, const char *path) { - return enabled > 0 && !is_absolute_path(path); + return cache->enabled > 0 && !is_absolute_path(path); } -/* - * Looks up a cache entry, waits if its being loaded by another thread. - * The mutex must be owned by the calling thread. - */ -static struct fsentry *fscache_get_wait(struct fsentry *key) +int fscache_enabled(const char *path) { - struct fsentry *fse = hashmap_get_entry(&map, key, ent, NULL); - - /* return if its a 'real' entry (future entries have refcnt == 0) */ - if (!fse || fse->list || fse->u.refcnt) - return fse; - - /* create an event and link our key to the future entry */ - key->u.hwait = CreateEvent(NULL, TRUE, FALSE, NULL); - key->next = fse->next; - fse->next = key; - - /* wait for the loading thread to signal us */ - LeaveCriticalSection(&mutex); - WaitForSingleObject(key->u.hwait, INFINITE); - CloseHandle(key->u.hwait); - EnterCriticalSection(&mutex); + struct fscache *cache = fscache_getcache(); - /* repeat cache lookup */ - return hashmap_get_entry(&map, key, ent, NULL); + return cache ? do_fscache_enabled(cache, path) : 0; } /* * Looks up or creates a cache entry for the specified key. */ -static struct fsentry *fscache_get(struct fsentry *key) +static struct fsentry *fscache_get(struct fscache *cache, struct fsentry *key) { - struct fsentry *fse, *future, *waiter; + struct fsentry *fse; int dir_not_found; - EnterCriticalSection(&mutex); - fscache_requests++; + cache->fscache_requests++; /* check if entry is in cache */ - fse = fscache_get_wait(key); + fse = hashmap_get_entry(&cache->map, key, ent, NULL); if (fse) { if (fse->st_mode) fsentry_addref(fse); else fse = NULL; /* non-existing directory */ - LeaveCriticalSection(&mutex); return fse; } /* if looking for a file, check if directory listing is in cache */ if (!fse && key->list) { - fse = fscache_get_wait(key->list); + fse = hashmap_get_entry(&cache->map, key->list, ent, NULL); if (fse) { - LeaveCriticalSection(&mutex); /* * dir entry without file entry, or dir does not * exist -> file doesn't exist @@ -349,25 +334,8 @@ static struct fsentry *fscache_get(struct fsentry *key) } } - /* add future entry to indicate that we're loading it */ - future = key->list ? key->list : key; - future->next = NULL; - future->u.refcnt = 0; - hashmap_add(&map, &future->ent); - - /* create the directory listing (outside mutex!) */ - LeaveCriticalSection(&mutex); - fse = fsentry_create_list(future, &dir_not_found); - EnterCriticalSection(&mutex); - - /* remove future entry and signal waiting threads */ - hashmap_remove(&map, &future->ent, NULL); - waiter = future->next; - while (waiter) { - HANDLE h = waiter->u.hwait; - waiter = waiter->next; - SetEvent(h); - } + /* create the directory listing */ + fse = fsentry_create_list(key->list ? key->list : key, &dir_not_found); /* leave on error (errno set by fsentry_create_list) */ if (!fse) { @@ -381,19 +349,18 @@ static struct fsentry *fscache_get(struct fsentry *key) key->list->dirent.d_name, key->list->len); fse->st_mode = 0; - hashmap_add(&map, &fse->ent); + hashmap_add(&cache->map, &fse->ent); } - LeaveCriticalSection(&mutex); return NULL; } /* add directory listing to the cache */ - fscache_misses++; - fscache_add(fse); + cache->fscache_misses++; + fscache_add(cache, fse); /* lookup file entry if requested (fse already points to directory) */ if (key->list) - fse = hashmap_get_entry(&map, key, ent, NULL); + fse = hashmap_get_entry(&cache->map, key, ent, NULL); if (fse && !fse->st_mode) fse = NULL; /* non-existing directory */ @@ -404,59 +371,104 @@ static struct fsentry *fscache_get(struct fsentry *key) else errno = ENOENT; - LeaveCriticalSection(&mutex); return fse; } /* - * Enables or disables the cache. Note that the cache is read-only, changes to + * Enables the cache. Note that the cache is read-only, changes to * the working directory are NOT reflected in the cache while enabled. */ -int fscache_enable(int enable, size_t initial_size) +int fscache_enable(size_t initial_size) { - int result; + int fscache; + struct fscache *cache; + int result = 0; + + /* allow the cache to be disabled entirely */ + fscache = git_env_bool("GIT_TEST_FSCACHE", -1); + if (fscache != -1) + core_fscache = fscache; + if (!core_fscache) + return 0; + /* + * refcount the global fscache initialization so that the + * opendir and lstat function pointers are redirected if + * any threads are using the fscache. + */ if (!initialized) { - int fscache = git_env_bool("GIT_TEST_FSCACHE", -1); - - /* allow the cache to be disabled entirely */ - if (fscache != -1) - core_fscache = fscache; - if (!core_fscache) - return 0; - InitializeCriticalSection(&mutex); - lstat_requests = opendir_requests = 0; - fscache_misses = fscache_requests = 0; + if (!dwTlsIndex) { + dwTlsIndex = TlsAlloc(); + if (dwTlsIndex == TLS_OUT_OF_INDEXES) { + LeaveCriticalSection(&mutex); + return 0; + } + } + + /* redirect opendir and lstat to the fscache implementations */ + opendir = fscache_opendir; + lstat = fscache_lstat; + } + InterlockedIncrement(&initialized); + + /* refcount the thread specific initialization */ + cache = fscache_getcache(); + if (cache) { + InterlockedIncrement(&cache->enabled); + } else { + cache = (struct fscache *)xcalloc(1, sizeof(*cache)); + cache->enabled = 1; /* * avoid having to rehash by leaving room for the parent dirs. * '4' was determined empirically by testing several repos */ - hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, initial_size * 4); - initialized = 1; + hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, initial_size * 4); + if (!TlsSetValue(dwTlsIndex, cache)) + BUG("TlsSetValue error"); } - result = enable ? InterlockedIncrement(&enabled) - : InterlockedDecrement(&enabled); + trace_printf_key(&trace_fscache, "fscache: enable\n"); + return result; +} - if (enable && result == 1) { - /* redirect opendir and lstat to the fscache implementations */ - opendir = fscache_opendir; - lstat = fscache_lstat; - } else if (!enable && !result) { +/* + * Disables the cache. + */ +void fscache_disable(void) +{ + struct fscache *cache; + + if (!core_fscache) + return; + + /* update the thread specific fscache initialization */ + cache = fscache_getcache(); + if (!cache) + BUG("fscache_disable() called on a thread where fscache has not been initialized"); + if (!cache->enabled) + BUG("fscache_disable() called on an fscache that is already disabled"); + InterlockedDecrement(&cache->enabled); + if (!cache->enabled) { + TlsSetValue(dwTlsIndex, NULL); + trace_printf_key(&trace_fscache, "fscache_disable: lstat %u, opendir %u, " + "total requests/misses %u/%u\n", + cache->lstat_requests, cache->opendir_requests, + cache->fscache_requests, cache->fscache_misses); + fscache_clear(cache); + free(cache); + } + + /* update the global fscache initialization */ + InterlockedDecrement(&initialized); + if (!initialized) { /* reset opendir and lstat to the original implementations */ opendir = dirent_opendir; lstat = mingw_lstat; - EnterCriticalSection(&mutex); - trace_printf_key(&trace_fscache, "fscache: lstat %u, opendir %u, " - "total requests/misses %u/%u\n", - lstat_requests, opendir_requests, - fscache_requests, fscache_misses); - fscache_clear(); - LeaveCriticalSection(&mutex); } - trace_printf_key(&trace_fscache, "fscache: enable(%d)\n", enable); - return result; + + trace_printf_key(&trace_fscache, "fscache: disable\n"); + return; } /* @@ -464,10 +476,10 @@ int fscache_enable(int enable, size_t initial_size) */ void fscache_flush(void) { - if (enabled) { - EnterCriticalSection(&mutex); - fscache_clear(); - LeaveCriticalSection(&mutex); + struct fscache *cache = fscache_getcache(); + + if (cache && cache->enabled) { + fscache_clear(cache); } } @@ -485,11 +497,12 @@ int fscache_lstat(const char *filename, struct stat *st) struct heap_fsentry key[2]; #pragma GCC diagnostic pop struct fsentry *fse; + struct fscache *cache = fscache_getcache(); - if (!fscache_enabled(filename)) + if (!cache || !do_fscache_enabled(cache, filename)) return mingw_lstat(filename, st); - lstat_requests++; + cache->lstat_requests++; /* split filename into path + name */ len = strlen(filename); if (len && is_dir_sep(filename[len - 1])) @@ -502,7 +515,7 @@ int fscache_lstat(const char *filename, struct stat *st) /* lookup entry for path + name in cache */ fsentry_init(&key[0].u.ent, NULL, filename, dirlen); fsentry_init(&key[1].u.ent, &key[0].u.ent, filename + base, len - base); - fse = fscache_get(&key[1].u.ent); + fse = fscache_get(cache, &key[1].u.ent); if (!fse) { errno = ENOENT; return -1; @@ -579,11 +592,12 @@ DIR *fscache_opendir(const char *dirname) struct fsentry *list; fscache_DIR *dir; int len; + struct fscache *cache = fscache_getcache(); - if (!fscache_enabled(dirname)) + if (!cache || !do_fscache_enabled(cache, dirname)) return dirent_opendir(dirname); - opendir_requests++; + cache->opendir_requests++; /* prepare name (strip trailing '/', replace '.') */ len = strlen(dirname); if ((len == 1 && dirname[0] == '.') || @@ -592,7 +606,7 @@ DIR *fscache_opendir(const char *dirname) /* get directory listing from cache */ fsentry_init(&key.u.ent, NULL, dirname, len); - list = fscache_get(&key.u.ent); + list = fscache_get(cache, &key.u.ent); if (!list) return NULL; @@ -603,3 +617,53 @@ DIR *fscache_opendir(const char *dirname) dir->pfsentry = list; return (DIR*) dir; } + +struct fscache *fscache_getcache(void) +{ + return (struct fscache *)TlsGetValue(dwTlsIndex); +} + +void fscache_merge(struct fscache *dest) +{ + struct hashmap_iter iter; + struct hashmap_entry *e; + struct fscache *cache = fscache_getcache(); + + /* + * Only do the merge if fscache was enabled and we have a dest + * cache to merge into. + */ + if (!dest) { + fscache_enable(0); + return; + } + if (!cache) + BUG("fscache_merge() called on a thread where fscache has not been initialized"); + + TlsSetValue(dwTlsIndex, NULL); + trace_printf_key(&trace_fscache, "fscache_merge: lstat %u, opendir %u, " + "total requests/misses %u/%u\n", + cache->lstat_requests, cache->opendir_requests, + cache->fscache_requests, cache->fscache_misses); + + /* + * This is only safe because the primary thread we're merging into + * isn't being used so the critical section only needs to prevent + * the the child threads from stomping on each other. + */ + EnterCriticalSection(&mutex); + + hashmap_iter_init(&cache->map, &iter); + while ((e = hashmap_iter_next(&iter))) + hashmap_add(&dest->map, e); + + dest->lstat_requests += cache->lstat_requests; + dest->opendir_requests += cache->opendir_requests; + dest->fscache_requests += cache->fscache_requests; + dest->fscache_misses += cache->fscache_misses; + LeaveCriticalSection(&mutex); + + free(cache); + + InterlockedDecrement(&initialized); +} diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h index d49c9381114da6..2eb8bf3f5cfee8 100644 --- a/compat/win32/fscache.h +++ b/compat/win32/fscache.h @@ -1,9 +1,16 @@ #ifndef FSCACHE_H #define FSCACHE_H -int fscache_enable(int enable, size_t initial_size); -#define enable_fscache(initial_size) fscache_enable(1, initial_size) -#define disable_fscache() fscache_enable(0, 0) +/* + * The fscache is thread specific. enable_fscache() must be called + * for each thread where caching is desired. + */ + +int fscache_enable(size_t initial_size); +#define enable_fscache(initial_size) fscache_enable(initial_size) + +void fscache_disable(void); +#define disable_fscache() fscache_disable() int fscache_enabled(const char *path); #define is_fscache_enabled(path) fscache_enabled(path) @@ -14,4 +21,13 @@ void fscache_flush(void); DIR *fscache_opendir(const char *dir); int fscache_lstat(const char *file_name, struct stat *buf); +/* opaque fscache structure */ +struct fscache; + +struct fscache *fscache_getcache(void); +#define getcache_fscache() fscache_getcache() + +void fscache_merge(struct fscache *dest); +#define merge_fscache(dest) fscache_merge(dest) + #endif diff --git a/git-compat-util.h b/git-compat-util.h index 0a1dc60210a4ae..10d3eec3c74ba5 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1096,6 +1096,10 @@ static inline int is_missing_file_error(int errno_) * data or even file content without the need to synchronize with the file * system. */ + + /* opaque fscache structure */ +struct fscache; + #ifndef enable_fscache #define enable_fscache(x) /* noop */ #endif @@ -1112,6 +1116,14 @@ static inline int is_missing_file_error(int errno_) #define flush_fscache() /* noop */ #endif +#ifndef getcache_fscache +#define getcache_fscache() (NULL) /* noop */ +#endif + +#ifndef merge_fscache +#define merge_fscache(dest) /* noop */ +#endif + int cmd_main(int, const char **); /* diff --git a/preload-index.c b/preload-index.c index e466fef15bcd79..ac0310008754a3 100644 --- a/preload-index.c +++ b/preload-index.c @@ -20,6 +20,8 @@ #include "trace2.h" #include "config.h" +static struct fscache *fscache; + /* * Mostly randomly chosen maximum thread counts: we * cap the parallelism to 20 threads, and we want @@ -57,6 +59,7 @@ static void *preload_thread(void *_data) nr = index->cache_nr - p->offset; last_nr = nr; + enable_fscache(nr); do { struct cache_entry *ce = *cep++; struct stat st; @@ -100,6 +103,7 @@ static void *preload_thread(void *_data) pthread_mutex_unlock(&pd->mutex); } cache_def_clear(&cache); + merge_fscache(fscache); return NULL; } @@ -118,6 +122,7 @@ void preload_index(struct index_state *index, if (!HAVE_THREADS || !core_preload_index) return; + fscache = getcache_fscache(); threads = index->cache_nr / THREAD_COST; if ((index->cache_nr > 1) && (threads < 2) && git_env_bool("GIT_TEST_PRELOAD_INDEX", 0)) threads = 2; @@ -141,7 +146,6 @@ void preload_index(struct index_state *index, pthread_mutex_init(&pd.mutex, NULL); } - enable_fscache(index->cache_nr); for (i = 0; i < threads; i++) { struct thread_data *p = data+i; int err; @@ -177,8 +181,6 @@ void preload_index(struct index_state *index, trace2_data_intmax("index", NULL, "preload/sum_lstat", t2_sum_lstat); trace2_region_leave("index", "preload", NULL); - - disable_fscache(); } int repo_read_index_preload(struct repository *repo, From 1c9be0407bb081d88e359ddb70ae043959ef7913 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Tue, 25 Sep 2018 16:28:16 -0400 Subject: [PATCH 184/229] fscache: add fscache hit statistics Track fscache hits and misses for lstat and opendir requests. Reporting of statistics is done when the cache is disabled for the last time and freed and is only reported if GIT_TRACE_FSCACHE is set. Sample output is: 11:33:11.836428 compat/win32/fscache.c:433 fscache: lstat 3775, opendir 263, total requests/misses 4052/269 Signed-off-by: Ben Peart --- compat/win32/fscache.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 935e36e5955c98..ad9fdd36964d93 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -11,6 +11,10 @@ static int initialized; static volatile long enabled; static struct hashmap map; static CRITICAL_SECTION mutex; +static unsigned int lstat_requests; +static unsigned int opendir_requests; +static unsigned int fscache_requests; +static unsigned int fscache_misses; static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); /* @@ -272,6 +276,8 @@ static void fscache_clear(void) { hashmap_clear_and_free(&map, struct fsentry, ent); hashmap_init(&map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); + lstat_requests = opendir_requests = 0; + fscache_misses = fscache_requests = 0; } /* @@ -318,6 +324,7 @@ static struct fsentry *fscache_get(struct fsentry *key) int dir_not_found; EnterCriticalSection(&mutex); + fscache_requests++; /* check if entry is in cache */ fse = fscache_get_wait(key); if (fse) { @@ -381,6 +388,7 @@ static struct fsentry *fscache_get(struct fsentry *key) } /* add directory listing to the cache */ + fscache_misses++; fscache_add(fse); /* lookup file entry if requested (fse already points to directory) */ @@ -418,6 +426,8 @@ int fscache_enable(int enable) return 0; InitializeCriticalSection(&mutex); + lstat_requests = opendir_requests = 0; + fscache_misses = fscache_requests = 0; hashmap_init(&map, (hashmap_cmp_fn) fsentry_cmp, NULL, 0); initialized = 1; } @@ -434,6 +444,10 @@ int fscache_enable(int enable) opendir = dirent_opendir; lstat = mingw_lstat; EnterCriticalSection(&mutex); + trace_printf_key(&trace_fscache, "fscache: lstat %u, opendir %u, " + "total requests/misses %u/%u\n", + lstat_requests, opendir_requests, + fscache_requests, fscache_misses); fscache_clear(); LeaveCriticalSection(&mutex); } @@ -471,6 +485,7 @@ int fscache_lstat(const char *filename, struct stat *st) if (!fscache_enabled(filename)) return mingw_lstat(filename, st); + lstat_requests++; /* split filename into path + name */ len = strlen(filename); if (len && is_dir_sep(filename[len - 1])) @@ -564,6 +579,7 @@ DIR *fscache_opendir(const char *dirname) if (!fscache_enabled(dirname)) return dirent_opendir(dirname); + opendir_requests++; /* prepare name (strip trailing '/', replace '.') */ len = strlen(dirname); if ((len == 1 && dirname[0] == '.') || From 156bffe8af837421b52c76f3b02358ddc246d667 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 11 Oct 2018 23:55:44 +0200 Subject: [PATCH 185/229] gitattributes: mark .png files as binary Signed-off-by: Johannes Schindelin --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 26490ad60a74d0..9d6e484d1ec0c4 100644 --- a/.gitattributes +++ b/.gitattributes @@ -6,6 +6,7 @@ *.pm text eol=lf diff=perl *.py text eol=lf diff=python *.bat text eol=crlf +*.png binary CODE_OF_CONDUCT.md -whitespace /Documentation/**/*.adoc text eol=lf whitespace=trail,space,incomplete /command-list.txt text eol=lf From 68d245860703a7a5a92d254314da2283e305f61d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 23 Apr 2018 23:20:00 +0200 Subject: [PATCH 186/229] fscache: Windows Docker volumes are *not* symbolic links ... even if they may look like them. As looking up the target of the "symbolic link" (just to see whether it starts with `/ContainerMappedDirectories/`) is pretty expensive, we do it when we can be *really* sure that there is a possibility that this might be the case. Signed-off-by: Johannes Schindelin Signed-off-by: JiSeop Moon --- compat/win32/fscache.c | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 46dca7a5635faf..efe6be7e5ef0a5 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -207,8 +207,30 @@ static struct fsentry *fseentry_create_entry(struct fscache *cache, fdata->FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT ? fdata->EaSize : 0; + /* + * On certain Windows versions, host directories mapped into + * Windows Containers ("Volumes", see https://docs.docker.com/storage/volumes/) + * look like symbolic links, but their targets are paths that + * are valid only in kernel mode. + * + * Let's work around this by detecting that situation and + * telling Git that these are *not* symbolic links. + */ + if (fse->reparse_tag == IO_REPARSE_TAG_SYMLINK && + sizeof(buf) > (size_t)(list ? list->len + 1 : 0) + fse->len + 1 && + is_inside_windows_container()) { + size_t off = 0; + if (list) { + memcpy(buf, list->dirent.d_name, list->len); + buf[list->len] = '/'; + off = list->len + 1; + } + memcpy(buf + off, fse->dirent.d_name, fse->len); + buf[off + fse->len] = '\0'; + } + fse->st_mode = file_attr_to_st_mode(fdata->FileAttributes, - fdata->EaSize); + fdata->EaSize, buf); fse->dirent.d_type = S_ISREG(fse->st_mode) ? DT_REG : S_ISDIR(fse->st_mode) ? DT_DIR : DT_LNK; fse->u.s.st_size = S_ISLNK(fse->st_mode) ? MAX_PATH : From 0c4baf790b150fda52787bc3cf8a5f858461f367 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Fri, 2 Nov 2018 11:19:10 -0400 Subject: [PATCH 187/229] fscache: teach fscache to use mempool Now that the fscache is single threaded, take advantage of the mem_pool as the allocator to significantly reduce the cost of allocations and frees. With the reduced cost of free, in future patches, we can start freeing the fscache at the end of commands instead of just leaking it. Signed-off-by: Ben Peart Signed-off-by: Johannes Schindelin --- compat/win32/fscache.c | 45 ++++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index cd57901aaf67b4..6da112e4f90382 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -6,6 +6,7 @@ #include "../../abspath.h" #include "../../trace.h" #include "config.h" +#include "../../mem-pool.h" static volatile long initialized; static DWORD dwTlsIndex; @@ -20,6 +21,7 @@ static CRITICAL_SECTION mutex; struct fscache { volatile long enabled; struct hashmap map; + struct mem_pool mem_pool; unsigned int lstat_requests; unsigned int opendir_requests; unsigned int fscache_requests; @@ -129,11 +131,12 @@ static void fsentry_init(struct fsentry *fse, struct fsentry *list, /* * Allocate an fsentry structure on the heap. */ -static struct fsentry *fsentry_alloc(struct fsentry *list, const char *name, +static struct fsentry *fsentry_alloc(struct fscache *cache, struct fsentry *list, const char *name, size_t len) { /* overallocate fsentry and copy the name to the end */ - struct fsentry *fse = xmalloc(sizeof(struct fsentry) + len + 1); + struct fsentry *fse = + mem_pool_alloc(&cache->mem_pool, sizeof(*fse) + len + 1); /* init the rest of the structure */ fsentry_init(fse, list, name, len); fse->next = NULL; @@ -153,27 +156,21 @@ inline static void fsentry_addref(struct fsentry *fse) } /* - * Release the reference to an fsentry, frees the memory if its the last ref. + * Release the reference to an fsentry. */ static void fsentry_release(struct fsentry *fse) { if (fse->list) fse = fse->list; - if (InterlockedDecrement(&(fse->u.refcnt))) - return; - - while (fse) { - struct fsentry *next = fse->next; - free(fse); - fse = next; - } + InterlockedDecrement(&(fse->u.refcnt)); } /* * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. */ -static struct fsentry *fseentry_create_entry(struct fsentry *list, +static struct fsentry *fseentry_create_entry(struct fscache *cache, + struct fsentry *list, const WIN32_FIND_DATAW *fdata) { char buf[MAX_PATH * 3]; @@ -181,7 +178,7 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, struct fsentry *fse; len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); - fse = fsentry_alloc(list, buf, len); + fse = fsentry_alloc(cache, list, buf, len); fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes, IO_REPARSE_TAG_SYMLINK); @@ -201,7 +198,7 @@ static struct fsentry *fseentry_create_entry(struct fsentry *list, * Dir should not contain trailing '/'. Use an empty string for the current * directory (not "."!). */ -static struct fsentry *fsentry_create_list(const struct fsentry *dir, +static struct fsentry *fsentry_create_list(struct fscache *cache, const struct fsentry *dir, int *dir_not_found) { wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ @@ -240,14 +237,14 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir, } /* allocate object to hold directory listing */ - list = fsentry_alloc(NULL, dir->dirent.d_name, dir->len); + list = fsentry_alloc(cache, NULL, dir->dirent.d_name, dir->len); list->st_mode = S_IFDIR; list->dirent.d_type = DT_DIR; /* walk directory and build linked list of fsentry structures */ phead = &list->next; do { - *phead = fseentry_create_entry(list, &fdata); + *phead = fseentry_create_entry(cache, list, &fdata); phead = &(*phead)->next; } while (FindNextFileW(h, &fdata)); @@ -259,7 +256,7 @@ static struct fsentry *fsentry_create_list(const struct fsentry *dir, if (err == ERROR_NO_MORE_FILES) return list; - /* otherwise free the list and return error */ + /* otherwise release the list and return error */ fsentry_release(list); errno = err_win_to_posix(err); return NULL; @@ -282,7 +279,9 @@ static void fscache_add(struct fscache *cache, struct fsentry *fse) */ static void fscache_clear(struct fscache *cache) { - hashmap_clear_and_free(&cache->map, struct fsentry, ent); + mem_pool_discard(&cache->mem_pool, 0); + mem_pool_init(&cache->mem_pool, 0); + hashmap_clear(&cache->map); hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, 0); cache->lstat_requests = cache->opendir_requests = 0; cache->fscache_misses = cache->fscache_requests = 0; @@ -335,7 +334,7 @@ static struct fsentry *fscache_get(struct fscache *cache, struct fsentry *key) } /* create the directory listing */ - fse = fsentry_create_list(key->list ? key->list : key, &dir_not_found); + fse = fsentry_create_list(cache, key->list ? key->list : key, &dir_not_found); /* leave on error (errno set by fsentry_create_list) */ if (!fse) { @@ -345,7 +344,7 @@ static struct fsentry *fscache_get(struct fscache *cache, struct fsentry *key) * empty, which for all practical matters is the same * thing as far as fscache is concerned). */ - fse = fsentry_alloc(key->list->list, + fse = fsentry_alloc(cache, key->list->list, key->list->dirent.d_name, key->list->len); fse->st_mode = 0; @@ -424,6 +423,7 @@ int fscache_enable(size_t initial_size) * '4' was determined empirically by testing several repos */ hashmap_init(&cache->map, (hashmap_cmp_fn)fsentry_cmp, NULL, initial_size * 4); + mem_pool_init(&cache->mem_pool, 0); if (!TlsSetValue(dwTlsIndex, cache)) BUG("TlsSetValue error"); } @@ -455,7 +455,8 @@ void fscache_disable(void) "total requests/misses %u/%u\n", cache->lstat_requests, cache->opendir_requests, cache->fscache_requests, cache->fscache_misses); - fscache_clear(cache); + mem_pool_discard(&cache->mem_pool, 0); + hashmap_clear(&cache->map); free(cache); } @@ -657,6 +658,8 @@ void fscache_merge(struct fscache *dest) while ((e = hashmap_iter_next(&iter))) hashmap_add(&dest->map, e); + mem_pool_combine(&dest->mem_pool, &cache->mem_pool); + dest->lstat_requests += cache->lstat_requests; dest->opendir_requests += cache->opendir_requests; dest->fscache_requests += cache->fscache_requests; From f7adccecb0f15683191bebff8c3b627d1814800f Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 12 Jun 2019 00:58:49 +0000 Subject: [PATCH 188/229] unpack-trees: enable fscache for sparse-checkout When updating the skip-worktree bits in the index to align with new values in a sparse-checkout file, Git scans the entire working directory with lstat() calls. In a sparse-checkout, many of these lstat() calls are for paths that do not exist. Enable the fscache feature during this scan. Since enable_fscache() calls nest, the disable_fscache() method decrements a counter and would only clear the cache if that counter reaches zero. In a local test of a repo with ~2.2 million paths, updating the index with git read-tree -m -u HEAD with a sparse-checkout file containing only /.gitattributes improved from 2-3 minutes to ~6 seconds. Signed-off-by: Derrick Stolee Signed-off-by: Johannes Schindelin --- unpack-trees.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unpack-trees.c b/unpack-trees.c index 1802809ad3c50a..949b2a4869ea99 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -1826,7 +1826,9 @@ static void mark_new_skip_worktree(struct pattern_list *pl, * 2. Widen worktree according to sparse-checkout file. * Matched entries will have skip_wt_flag cleared (i.e. "in") */ + enable_fscache(istate->cache_nr); clear_ce_flags(istate, select_flag, skip_wt_flag, pl, show_progress); + enable_fscache(0); } static void populate_from_existing_patterns(struct unpack_trees_options *o, From af0da6e071f52c24a9fdc1b846534d0236196896 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 5 Aug 2017 20:28:37 +0200 Subject: [PATCH 189/229] tests: move test PNGs into t/lib-diff/ We already have a directory where we store files intended for use by multiple test scripts. The same directory is a better home for the test-binary-*.png files than t/. Signed-off-by: Johannes Schindelin --- t/{ => lib-diff}/test-binary-1.png | Bin t/{ => lib-diff}/test-binary-2.png | Bin t/t3307-notes-man.sh | 2 +- t/t3903-stash.sh | 2 +- t/t4012-diff-binary.sh | 2 +- t/t4049-diff-stat-count.sh | 2 +- t/t4108-apply-threeway.sh | 12 ++++++------ t/t6403-merge-file.sh | 4 ++-- t/t6407-merge-binary.sh | 2 +- t/t9200-git-cvsexportcommit.sh | 14 +++++++------- 10 files changed, 20 insertions(+), 20 deletions(-) rename t/{ => lib-diff}/test-binary-1.png (100%) rename t/{ => lib-diff}/test-binary-2.png (100%) diff --git a/t/test-binary-1.png b/t/lib-diff/test-binary-1.png similarity index 100% rename from t/test-binary-1.png rename to t/lib-diff/test-binary-1.png diff --git a/t/test-binary-2.png b/t/lib-diff/test-binary-2.png similarity index 100% rename from t/test-binary-2.png rename to t/lib-diff/test-binary-2.png diff --git a/t/t3307-notes-man.sh b/t/t3307-notes-man.sh index 1aa366a410e9a3..7e5c06e6615d7a 100755 --- a/t/t3307-notes-man.sh +++ b/t/t3307-notes-man.sh @@ -26,7 +26,7 @@ test_expect_success 'example 1: notes to add an Acked-by line' ' ' test_expect_success 'example 2: binary notes' ' - cp "$TEST_DIRECTORY"/test-binary-1.png . && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png . && git checkout B && blob=$(git hash-object -w test-binary-1.png) && git notes --ref=logo add -C "$blob" && diff --git a/t/t3903-stash.sh b/t/t3903-stash.sh index 70c60319588190..a5933528d24be9 100755 --- a/t/t3903-stash.sh +++ b/t/t3903-stash.sh @@ -1437,7 +1437,7 @@ test_expect_success 'stash -- works with binary files' ' mkdir -p subdir && >subdir/untracked && >subdir/tracked && - cp "$TEST_DIRECTORY"/test-binary-1.png subdir/tracked-binary && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png subdir/tracked-binary && git add subdir/tracked* && git stash -- subdir/ && test_path_is_missing subdir/tracked && diff --git a/t/t4012-diff-binary.sh b/t/t4012-diff-binary.sh index 97b5ac04071d36..0fb50d2ffc91d9 100755 --- a/t/t4012-diff-binary.sh +++ b/t/t4012-diff-binary.sh @@ -19,7 +19,7 @@ test_expect_success 'prepare repository' ' echo AIT >a && echo BIT >b && echo CIT >c && echo DIT >d && git update-index --add a b c d && echo git >a && - cat "$TEST_DIRECTORY"/test-binary-1.png >b && + cat "$TEST_DIRECTORY"/lib-diff/test-binary-1.png >b && echo git >c && cat b b >d ' diff --git a/t/t4049-diff-stat-count.sh b/t/t4049-diff-stat-count.sh index eceb47c8594416..2161a1e8cf5ba6 100755 --- a/t/t4049-diff-stat-count.sh +++ b/t/t4049-diff-stat-count.sh @@ -33,7 +33,7 @@ test_expect_success 'binary changes do not count in lines' ' git reset --hard && echo a >a && echo c >c && - cat "$TEST_DIRECTORY"/test-binary-1.png >d && + cat "$TEST_DIRECTORY"/lib-diff/test-binary-1.png >d && cat >expect <<-\EOF && a | 1 + c | 1 + diff --git a/t/t4108-apply-threeway.sh b/t/t4108-apply-threeway.sh index f30e85659dbb87..7f84edd9653a7d 100755 --- a/t/t4108-apply-threeway.sh +++ b/t/t4108-apply-threeway.sh @@ -272,11 +272,11 @@ test_expect_success 'apply with --3way --cached and conflicts' ' test_expect_success 'apply binary file patch' ' git reset --hard main && - cp "$TEST_DIRECTORY/test-binary-1.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-1.png" bin.png && git add bin.png && git commit -m "add binary file" && - cp "$TEST_DIRECTORY/test-binary-2.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-2.png" bin.png && git diff --binary >bin.diff && git reset --hard && @@ -287,11 +287,11 @@ test_expect_success 'apply binary file patch' ' test_expect_success 'apply binary file patch with 3way' ' git reset --hard main && - cp "$TEST_DIRECTORY/test-binary-1.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-1.png" bin.png && git add bin.png && git commit -m "add binary file" && - cp "$TEST_DIRECTORY/test-binary-2.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-2.png" bin.png && git diff --binary >bin.diff && git reset --hard && @@ -302,11 +302,11 @@ test_expect_success 'apply binary file patch with 3way' ' test_expect_success 'apply full-index patch with 3way' ' git reset --hard main && - cp "$TEST_DIRECTORY/test-binary-1.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-1.png" bin.png && git add bin.png && git commit -m "add binary file" && - cp "$TEST_DIRECTORY/test-binary-2.png" bin.png && + cp "$TEST_DIRECTORY/lib-diff/test-binary-2.png" bin.png && git diff --full-index >bin.diff && git reset --hard && diff --git a/t/t6403-merge-file.sh b/t/t6403-merge-file.sh index dcc5c2ac99e7f1..bd58e471bc724a 100755 --- a/t/t6403-merge-file.sh +++ b/t/t6403-merge-file.sh @@ -355,12 +355,12 @@ test_expect_success "expected conflict markers" ' test_expect_success 'binary files cannot be merged' ' test_must_fail git merge-file -p \ - orig.txt "$TEST_DIRECTORY"/test-binary-1.png new1.txt 2> merge.err && + orig.txt "$TEST_DIRECTORY"/lib-diff/test-binary-1.png new1.txt 2> merge.err && test_grep "Cannot merge binary files" merge.err ' test_expect_success 'binary files cannot be merged with --object-id' ' - cp "$TEST_DIRECTORY"/test-binary-1.png . && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png . && git add orig.txt new1.txt test-binary-1.png && test_must_fail git merge-file --object-id \ :orig.txt :test-binary-1.png :new1.txt 2> merge.err && diff --git a/t/t6407-merge-binary.sh b/t/t6407-merge-binary.sh index e8a28717cece32..2547f1d504a2c5 100755 --- a/t/t6407-merge-binary.sh +++ b/t/t6407-merge-binary.sh @@ -9,7 +9,7 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME test_expect_success setup ' - cat "$TEST_DIRECTORY"/test-binary-1.png >m && + cat "$TEST_DIRECTORY"/lib-diff/test-binary-1.png >m && git add m && git ls-files -s | sed -e "s/ 0 / 1 /" >E1 && test_tick && diff --git a/t/t9200-git-cvsexportcommit.sh b/t/t9200-git-cvsexportcommit.sh index 8e0272df20fd2a..c42e69ee70ee23 100755 --- a/t/t9200-git-cvsexportcommit.sh +++ b/t/t9200-git-cvsexportcommit.sh @@ -57,8 +57,8 @@ test_expect_success 'New file' ' mkdir A B C D E F && echo hello1 >A/newfile1.txt && echo hello2 >B/newfile2.txt && - cp "$TEST_DIRECTORY"/test-binary-1.png C/newfile3.png && - cp "$TEST_DIRECTORY"/test-binary-1.png D/newfile4.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png C/newfile3.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png D/newfile4.png && git add A/newfile1.txt && git add B/newfile2.txt && git add C/newfile3.png && @@ -83,8 +83,8 @@ test_expect_success 'Remove two files, add two and update two' ' rm -f B/newfile2.txt && rm -f C/newfile3.png && echo Hello5 >E/newfile5.txt && - cp "$TEST_DIRECTORY"/test-binary-2.png D/newfile4.png && - cp "$TEST_DIRECTORY"/test-binary-1.png F/newfile6.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-2.png D/newfile4.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png F/newfile6.png && git add E/newfile5.txt && git add F/newfile6.png && git commit -a -m "Test: Remove, add and update" && @@ -172,7 +172,7 @@ test_expect_success 'New file with spaces in file name' ' mkdir "G g" && echo ok then >"G g/with spaces.txt" && git add "G g/with spaces.txt" && \ - cp "$TEST_DIRECTORY"/test-binary-1.png "G g/with spaces.png" && \ + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png "G g/with spaces.png" && \ git add "G g/with spaces.png" && git commit -a -m "With spaces" && id=$(git rev-list --max-count=1 HEAD) && @@ -184,7 +184,7 @@ test_expect_success 'New file with spaces in file name' ' test_expect_success 'Update file with spaces in file name' ' echo Ok then >>"G g/with spaces.txt" && - cat "$TEST_DIRECTORY"/test-binary-1.png >>"G g/with spaces.png" && \ + cat "$TEST_DIRECTORY"/lib-diff/test-binary-1.png >>"G g/with spaces.png" && \ git add "G g/with spaces.png" && git commit -a -m "Update with spaces" && id=$(git rev-list --max-count=1 HEAD) && @@ -209,7 +209,7 @@ test_expect_success !MINGW 'File with non-ascii file name' ' mkdir -p Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö && echo Foo >Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.txt && git add Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.txt && - cp "$TEST_DIRECTORY"/test-binary-1.png Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.png && + cp "$TEST_DIRECTORY"/lib-diff/test-binary-1.png Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.png && git add Å/goo/a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/q/r/s/t/u/v/w/x/y/z/å/ä/ö/gårdetsågårdet.png && git commit -a -m "Går det så går det" && \ id=$(git rev-list --max-count=1 HEAD) && From 3f3e6316a004414ee9ae8cd301b435aa3012b876 Mon Sep 17 00:00:00 2001 From: xungeng li Date: Wed, 7 Jun 2023 20:26:33 +0800 Subject: [PATCH 190/229] fscache: optionally enable wsl compability file mode bits The Windows Subsystem for Linux (WSL) version 2 allows to use `chmod` on NTFS volumes provided that they are mounted with metadata enabled (see https://devblogs.microsoft.com/commandline/chmod-chown-wsl-improvements/ for details), for example: $ chmod 0755 /mnt/d/test/a.sh In order to facilitate better collaboration between the Windows version of Git and the WSL version of Git, we can make the Windows version of Git also support reading and writing NTFS file modes in a manner compatible with WSL. Since this slightly slows down operations where lots of files are created (such as an initial checkout), this feature is only enabled when `core.WSLCompat` is set to true. Note that you also have to set `core.fileMode=true` in repositories that have been initialized without enabling WSL compatibility. There are several ways to enable metadata loading for NTFS volumes in WSL, one of which is to modify `/etc/wsl.conf` by adding: ``` [automount] enabled = true options = "metadata,umask=027,fmask=117" ``` And reboot WSL. It can also be enabled temporarily by this incantation: $ sudo umount /mnt/c && sudo mount -t drvfs C: /mnt/c -o metadata,uid=1000,gid=1000,umask=22,fmask=111 It's important to note that this modification is compatible with, but does not depend on WSL. The helper functions in this commit can operate independently and functions normally on devices where WSL is not installed or properly configured. Signed-off-by: xungeng li Signed-off-by: Johannes Schindelin --- compat/win32/fscache.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index efe6be7e5ef0a5..e247e9a6663131 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -8,6 +8,7 @@ #include "config.h" #include "../../mem-pool.h" #include "ntifs.h" +#include "wsl.h" static volatile long initialized; static DWORD dwTlsIndex; @@ -242,6 +243,21 @@ static struct fsentry *fseentry_create_entry(struct fscache *cache, &(fse->u.s.st_mtim)); filetime_to_timespec((FILETIME *)&(fdata->CreationTime), &(fse->u.s.st_ctim)); + if (fdata->EaSize > 0 && + sizeof(buf) >= (size_t)(list ? list->len+1 : 0) + fse->len+1 && + are_wsl_compatible_mode_bits_enabled()) { + size_t off = 0; + wchar_t wpath[MAX_LONG_PATH]; + if (list && list->len) { + memcpy(buf, list->dirent.d_name, list->len); + buf[list->len] = '/'; + off = list->len + 1; + } + memcpy(buf + off, fse->dirent.d_name, fse->len); + buf[off + fse->len] = '\0'; + if (xutftowcs_long_path(wpath, buf) >= 0) + copy_wsl_mode_bits_from_disk(wpath, -1, &fse->st_mode); + } return fse; } From 08caed8bf5c725f5a00c76eed007093cb7a7764c Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Fri, 16 Nov 2018 10:59:18 -0500 Subject: [PATCH 191/229] fscache: make fscache_enable() thread safe The recent change to make fscache thread specific relied on fscache_enable() being called first from the primary thread before being called in parallel from worker threads. Make that more robust and protect it with a critical section to avoid any issues. Helped-by: Johannes Schindelin Signed-off-by: Ben Peart --- compat/mingw.c | 4 ++++ compat/win32/fscache.c | 23 +++++++++++++---------- compat/win32/fscache.h | 2 ++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/compat/mingw.c b/compat/mingw.c index 76fa68348fcf6d..8d667655244372 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -17,6 +17,7 @@ #include "trace2.h" #include "win32.h" #include "win32/exit-process.h" +#include "win32/fscache.h" #include "win32/lazyload.h" #include "win32/wsl.h" #include "wrapper.h" @@ -4289,6 +4290,9 @@ int wmain(int argc, const wchar_t **wargv) InitializeCriticalSection(&pinfo_cs); InitializeCriticalSection(&phantom_symlinks_cs); + /* initialize critical section for fscache */ + InitializeCriticalSection(&fscache_cs); + /* set up default file mode and file modes for stdin/out/err */ _fmode = _O_BINARY; _setmode(_fileno(stdin), _O_BINARY); diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 6da112e4f90382..335c46dd36c544 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -10,7 +10,7 @@ static volatile long initialized; static DWORD dwTlsIndex; -static CRITICAL_SECTION mutex; +CRITICAL_SECTION fscache_cs; /* * Store one fscache per thread to avoid thread contention and locking. @@ -395,12 +395,12 @@ int fscache_enable(size_t initial_size) * opendir and lstat function pointers are redirected if * any threads are using the fscache. */ + EnterCriticalSection(&fscache_cs); if (!initialized) { - InitializeCriticalSection(&mutex); if (!dwTlsIndex) { dwTlsIndex = TlsAlloc(); if (dwTlsIndex == TLS_OUT_OF_INDEXES) { - LeaveCriticalSection(&mutex); + LeaveCriticalSection(&fscache_cs); return 0; } } @@ -409,12 +409,13 @@ int fscache_enable(size_t initial_size) opendir = fscache_opendir; lstat = fscache_lstat; } - InterlockedIncrement(&initialized); + initialized++; + LeaveCriticalSection(&fscache_cs); /* refcount the thread specific initialization */ cache = fscache_getcache(); if (cache) { - InterlockedIncrement(&cache->enabled); + cache->enabled++; } else { cache = (struct fscache *)xcalloc(1, sizeof(*cache)); cache->enabled = 1; @@ -448,7 +449,7 @@ void fscache_disable(void) BUG("fscache_disable() called on a thread where fscache has not been initialized"); if (!cache->enabled) BUG("fscache_disable() called on an fscache that is already disabled"); - InterlockedDecrement(&cache->enabled); + cache->enabled--; if (!cache->enabled) { TlsSetValue(dwTlsIndex, NULL); trace_printf_key(&trace_fscache, "fscache_disable: lstat %u, opendir %u, " @@ -461,12 +462,14 @@ void fscache_disable(void) } /* update the global fscache initialization */ - InterlockedDecrement(&initialized); + EnterCriticalSection(&fscache_cs); + initialized--; if (!initialized) { /* reset opendir and lstat to the original implementations */ opendir = dirent_opendir; lstat = mingw_lstat; } + LeaveCriticalSection(&fscache_cs); trace_printf_key(&trace_fscache, "fscache: disable\n"); return; @@ -652,7 +655,7 @@ void fscache_merge(struct fscache *dest) * isn't being used so the critical section only needs to prevent * the the child threads from stomping on each other. */ - EnterCriticalSection(&mutex); + EnterCriticalSection(&fscache_cs); hashmap_iter_init(&cache->map, &iter); while ((e = hashmap_iter_next(&iter))) @@ -664,9 +667,9 @@ void fscache_merge(struct fscache *dest) dest->opendir_requests += cache->opendir_requests; dest->fscache_requests += cache->fscache_requests; dest->fscache_misses += cache->fscache_misses; - LeaveCriticalSection(&mutex); + initialized--; + LeaveCriticalSection(&fscache_cs); free(cache); - InterlockedDecrement(&initialized); } diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h index 2eb8bf3f5cfee8..042b247a542554 100644 --- a/compat/win32/fscache.h +++ b/compat/win32/fscache.h @@ -6,6 +6,8 @@ * for each thread where caching is desired. */ +extern CRITICAL_SECTION fscache_cs; + int fscache_enable(size_t initial_size); #define enable_fscache(initial_size) fscache_enable(initial_size) From c6c96005a30cdea90a1b5c3037de9799b236f084 Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 1 Nov 2018 11:40:51 -0400 Subject: [PATCH 192/229] status: disable and free fscache at the end of the status command At the end of the status command, disable and free the fscache so that we don't leak the memory and so that we can dump the fscache statistics. Signed-off-by: Ben Peart --- builtin/commit.c | 1 + 1 file changed, 1 insertion(+) diff --git a/builtin/commit.c b/builtin/commit.c index 0baee46574e543..2eccf5d2f11fff 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1666,6 +1666,7 @@ struct repository *repo UNUSED) wt_status_print(&s); wt_status_collect_free_buffers(&s); + enable_fscache(0); return 0; } From 1232d9ddf976e9caece3bda3d62c203452dfb7d9 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 18 Jul 2017 01:15:40 +0200 Subject: [PATCH 193/229] tests: only override sort & find if there are usable ones in /usr/bin/ The idea is to allow running the test suite on MinGit with BusyBox installed in /mingw64/bin/sh.exe. In that case, we will want to exclude sort & find (and other Unix utilities) from being bundled. Signed-off-by: Johannes Schindelin --- git-sh-setup.sh | 21 ++++++++++++++------- t/test-lib.sh | 21 ++++++++++++++------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/git-sh-setup.sh b/git-sh-setup.sh index 19aef72ec25530..fad4f9df94e143 100644 --- a/git-sh-setup.sh +++ b/git-sh-setup.sh @@ -292,13 +292,20 @@ create_virtual_base() { # Platform specific tweaks to work around some commands case $(uname -s) in *MINGW*) - # Windows has its own (incompatible) sort and find - sort () { - /usr/bin/sort "$@" - } - find () { - /usr/bin/find "$@" - } + if test -x /usr/bin/sort + then + # Windows has its own (incompatible) sort; override + sort () { + /usr/bin/sort "$@" + } + fi + if test -x /usr/bin/find + then + # Windows has its own (incompatible) find; override + find () { + /usr/bin/find "$@" + } + fi # git sees Windows-style pwd pwd () { builtin pwd -W diff --git a/t/test-lib.sh b/t/test-lib.sh index 11dc2be5070ebb..37af8a3037b086 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1712,13 +1712,20 @@ Darwin) test_set_prereq EXECKEEPSPID ;; *MINGW*) - # Windows has its own (incompatible) sort and find - sort () { - /usr/bin/sort "$@" - } - find () { - /usr/bin/find "$@" - } + if test -x /usr/bin/sort + then + # Windows has its own (incompatible) sort; override + sort () { + /usr/bin/sort "$@" + } + fi + if test -x /usr/bin/find + then + # Windows has its own (incompatible) find; override + find () { + /usr/bin/find "$@" + } + fi # git sees Windows-style pwd pwd () { builtin pwd -W From 99c8a4711266d33391835d4508665c08fbe288c2 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Tue, 1 Sep 2026 11:38:25 -0700 Subject: [PATCH 194/229] compat/poll: do not collect more handles than the wait supports The Windows implementation of poll() collects one wait handle per polled descriptor in HANDLE h, handle_array[FD_SETSIZE + 2]; and appends to it without a bounds check. It then writes a NULL sentinel at handle_array[nhandles]. A caller with enough live descriptors therefore writes past the end of the array and corrupts the stack. The corruption is silent, and when it reaches the stack cookie the process aborts with STATUS_STACK_BUFFER_OVERRUN. The array is not the only limit. The collected handles are passed to MsgWaitForMultipleObjects (nhandles, handle_array, FALSE, wait_timeout, QS_ALLINPUT); which waits on at most MAXIMUM_WAIT_OBJECTS objects, and QS_ALLINPUT adds the thread message queue as one more object beyond the handles. The code shows this, because it reports the message queue as WAIT_OBJECT_0 + nhandles. One further handle is poll()'s own event object. So at most MAXIMUM_WAIT_OBJECTS - 2 descriptors can be waited on, which is the tighter of the two bounds and is well inside the array. Define that limit as POLL_MAX_DESCRIPTORS next to the poll() declaration, and refuse to collect beyond it, returning EINVAL. Two preprocessor checks tie the constant to MAXIMUM_WAIT_OBJECTS and to the size of handle_array, so the two cannot drift apart. poll() is now memory-safe for every input, and a case that previously smashed the stack fails cleanly. Undo the WSAEventSelect() registrations before returning. The loop that normally does this runs after the wait, and the new error path skips it, which would otherwise leave those sockets associated with poll()'s static event object and let later socket activity disturb an unrelated poll(). Note that the limit is on the number of handles actually collected, not on nfd. Those are different: a descriptor only takes a handle when it is non-negative, is not a socket, and has no events pending yet. Sockets are all multiplexed onto the one event object. Callers routinely pass sparse arrays, for example run_processes_parallel(), which sizes its pollfd array to the configured job count and leaves the unused slots at fd = -1. Rejecting a large nfd would break such callers even though they never come close to the wait limit. For platforms with a native poll(), which has no such limit, define POLL_MAX_DESCRIPTORS to INT_MAX so that callers can clamp against it unconditionally. Signed-off-by: Tyrie Vella --- Makefile | 1 + compat/poll/poll.c | 48 ++++++++++++- compat/poll/poll.h | 16 +++++ compat/posix.h | 10 +++ t/meson.build | 1 + t/unit-tests/u-poll.c | 163 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 238 insertions(+), 1 deletion(-) create mode 100644 t/unit-tests/u-poll.c diff --git a/Makefile b/Makefile index 34c946386e7b6c..f5e5d6a04de958 100644 --- a/Makefile +++ b/Makefile @@ -1565,6 +1565,7 @@ CLAR_TEST_SUITES += u-odb-inmemory CLAR_TEST_SUITES += u-oid-array CLAR_TEST_SUITES += u-oidmap CLAR_TEST_SUITES += u-oidtree +CLAR_TEST_SUITES += u-poll CLAR_TEST_SUITES += u-prio-queue CLAR_TEST_SUITES += u-reftable-basics CLAR_TEST_SUITES += u-reftable-block diff --git a/compat/poll/poll.c b/compat/poll/poll.c index ea362b4a8e2340..1205f09ce9fbc8 100644 --- a/compat/poll/poll.c +++ b/compat/poll/poll.c @@ -303,6 +303,43 @@ compute_revents (int fd, int sought, fd_set *rfds, fd_set *wfds, fd_set *efds) } #endif /* !MinGW */ +#ifdef WIN32_NATIVE +/* POLL_MAX_DESCRIPTORS descriptors, plus hEvent and the QS_ALLINPUT message + queue, must fit in one MsgWaitForMultipleObjects call, and the collected + handles plus the NULL sentinel must fit in handle_array. */ +#if POLL_MAX_DESCRIPTORS + 2 > MAXIMUM_WAIT_OBJECTS +#error POLL_MAX_DESCRIPTORS exceeds MAXIMUM_WAIT_OBJECTS +#endif +#if POLL_MAX_DESCRIPTORS + 2 > FD_SETSIZE + 2 +#error POLL_MAX_DESCRIPTORS does not fit in handle_array +#endif + +/* Undo the WSAEventSelect() calls made for the first NFD descriptors. */ +static void +reset_socket_events (struct pollfd *pfd, nfds_t nfd) +{ + nfds_t i; + + for (i = 0; i < nfd; i++) + { + HANDLE h; + + if (pfd[i].fd < 0) + continue; + if (!(pfd[i].events & (POLLIN | POLLRDNORM | POLLOUT | POLLWRNORM | + POLLWRBAND | POLLPRI | POLLRDBAND))) + continue; + + h = (HANDLE) _get_osfhandle (pfd[i].fd); + if (h == NULL || h == INVALID_HANDLE_VALUE) + continue; + + if (IsSocketHandle (h)) + WSAEventSelect ((SOCKET) h, NULL, 0); + } +} +#endif + int poll (struct pollfd *pfd, nfds_t nfd, int timeout) { @@ -504,7 +541,16 @@ poll (struct pollfd *pfd, nfds_t nfd, int timeout) bits for the "wrong" direction. */ pfd[i].revents = win32_compute_revents (h, &sought); if (sought) - handle_array[nhandles++] = h; + { + /* hEvent occupies handle_array[0]. See POLL_MAX_DESCRIPTORS. */ + if (nhandles > POLL_MAX_DESCRIPTORS) + { + reset_socket_events (pfd, i); + errno = EINVAL; + return -1; + } + handle_array[nhandles++] = h; + } if (pfd[i].revents) timeout = 0; } diff --git a/compat/poll/poll.h b/compat/poll/poll.h index 1e1597360f4485..5c1169acc2c003 100644 --- a/compat/poll/poll.h +++ b/compat/poll/poll.h @@ -59,6 +59,22 @@ typedef unsigned long nfds_t; extern int poll (struct pollfd *pfd, nfds_t nfd, int timeout); +#if (defined _WIN32 || defined __WIN32__) && ! defined __CYGWIN__ +/* + * This poll() is emulated with MsgWaitForMultipleObjects(), which waits on at + * most MAXIMUM_WAIT_OBJECTS (64) objects. Two of those are never available for + * polled descriptors: poll() waits on its own event object, and QS_ALLINPUT + * adds the thread message queue. Sockets do not count, because they are all + * multiplexed onto that one event object; every other descriptor takes a wait + * slot of its own. + * + * Callers that poll one or more descriptors per child must keep the number of + * simultaneously live descriptors within this limit. Exceeding it fails with + * EINVAL. + */ +#define POLL_MAX_DESCRIPTORS 62 +#endif + /* Define INFTIM only if doing so conforms to POSIX. */ #if !defined (_POSIX_C_SOURCE) && !defined (_XOPEN_SOURCE) #define INFTIM (-1) diff --git a/compat/posix.h b/compat/posix.h index 108d38e0df58d7..73d01c42446499 100644 --- a/compat/posix.h +++ b/compat/posix.h @@ -133,6 +133,16 @@ /* Pull the compat stuff */ #include #endif + +/* + * compat/poll defines POLL_MAX_DESCRIPTORS to the largest number of + * descriptors its poll() emulation can wait on. A native poll() has no such + * limit, so callers that fan out one descriptor per child can clamp against + * this unconditionally. + */ +#ifndef POLL_MAX_DESCRIPTORS +#define POLL_MAX_DESCRIPTORS INT_MAX +#endif #ifdef HAVE_BSD_SYSCTL #include #endif diff --git a/t/meson.build b/t/meson.build index ff39f526c22f61..e352ca353a09d5 100644 --- a/t/meson.build +++ b/t/meson.build @@ -11,6 +11,7 @@ clar_test_suites = [ 'unit-tests/u-oid-array.c', 'unit-tests/u-oidmap.c', 'unit-tests/u-oidtree.c', + 'unit-tests/u-poll.c', 'unit-tests/u-prio-queue.c', 'unit-tests/u-reftable-basics.c', 'unit-tests/u-reftable-block.c', diff --git a/t/unit-tests/u-poll.c b/t/unit-tests/u-poll.c new file mode 100644 index 00000000000000..bc72135da28451 --- /dev/null +++ b/t/unit-tests/u-poll.c @@ -0,0 +1,163 @@ +#include "unit-test.h" + +#ifdef GIT_WINDOWS_NATIVE +static struct { + int pipes[POLL_MAX_DESCRIPTORS + 1][2]; + size_t nr_pipes; + int listener; + int sockets[2]; + WSAEVENT event; + int event_selected; +} poll_test; + +static void open_pipes(struct pollfd *fds, size_t nr) +{ + size_t i; + + cl_assert(nr <= ARRAY_SIZE(poll_test.pipes)); + for (i = 0; i < nr; i++) { + int *pipefd = poll_test.pipes[poll_test.nr_pipes]; + + cl_assert_equal_i(pipe(pipefd), 0); + poll_test.nr_pipes++; + fds[i].fd = pipefd[0]; + fds[i].events = POLLIN; + } +} + +static void create_socket_pair(void) +{ + struct sockaddr_in address = { + .sin_family = AF_INET, + .sin_addr.s_addr = htonl(INADDR_LOOPBACK), + }; + socklen_t address_length = sizeof(address); + + poll_test.listener = socket(AF_INET, SOCK_STREAM, 0); + cl_assert(poll_test.listener >= 0); + cl_assert_equal_i(bind(poll_test.listener, + (struct sockaddr *)&address, + sizeof(address)), 0); + cl_assert_equal_i(getsockname( + (SOCKET)_get_osfhandle(poll_test.listener), + (struct sockaddr *)&address, + &address_length), 0); + cl_assert_equal_i(listen(poll_test.listener, 1), 0); + + poll_test.sockets[0] = socket(AF_INET, SOCK_STREAM, 0); + cl_assert(poll_test.sockets[0] >= 0); + cl_assert_equal_i(connect(poll_test.sockets[0], + (struct sockaddr *)&address, + sizeof(address)), 0); + + poll_test.sockets[1] = accept(poll_test.listener, NULL, NULL); + cl_assert(poll_test.sockets[1] >= 0); + close(poll_test.listener); + poll_test.listener = -1; +} +#endif + +void test_poll__initialize(void) +{ +#ifdef GIT_WINDOWS_NATIVE + memset(&poll_test, 0, sizeof(poll_test)); + poll_test.listener = -1; + poll_test.sockets[0] = -1; + poll_test.sockets[1] = -1; + poll_test.event = WSA_INVALID_EVENT; +#endif +} + +void test_poll__cleanup(void) +{ +#ifdef GIT_WINDOWS_NATIVE + size_t i; + + if (poll_test.event_selected && poll_test.sockets[0] >= 0) + WSAEventSelect( + (SOCKET)_get_osfhandle(poll_test.sockets[0]), NULL, 0); + if (poll_test.event != WSA_INVALID_EVENT) + WSACloseEvent(poll_test.event); + if (poll_test.listener >= 0) + close(poll_test.listener); + for (i = 0; i < ARRAY_SIZE(poll_test.sockets); i++) + if (poll_test.sockets[i] >= 0) + close(poll_test.sockets[i]); + for (i = 0; i < poll_test.nr_pipes; i++) { + close(poll_test.pipes[i][0]); + close(poll_test.pipes[i][1]); + } +#endif +} + +void test_poll__limit(void) +{ +#ifdef GIT_WINDOWS_NATIVE + struct pollfd fds[POLL_MAX_DESCRIPTORS + 1] = { 0 }; + + open_pipes(fds, ARRAY_SIZE(fds)); + cl_assert_equal_i(poll(fds, POLL_MAX_DESCRIPTORS, 0), 0); + + errno = 0; + cl_assert_equal_i(poll(fds, ARRAY_SIZE(fds), 0), -1); + cl_assert_equal_i(errno, EINVAL); +#else + cl_skip(); +#endif +} + +void test_poll__sparse(void) +{ +#ifdef GIT_WINDOWS_NATIVE + struct pollfd fds[2 * POLL_MAX_DESCRIPTORS + 1] = { 0 }; + size_t i; + + open_pipes(fds, POLL_MAX_DESCRIPTORS); + create_socket_pair(); + + for (i = POLL_MAX_DESCRIPTORS; i > 0; i--) { + fds[2 * i - 1] = fds[i - 1]; + fds[2 * i - 2].fd = -1; + } + fds[2 * POLL_MAX_DESCRIPTORS].fd = poll_test.sockets[0]; + fds[2 * POLL_MAX_DESCRIPTORS].events = POLLIN; + + cl_assert_equal_i(poll(fds, ARRAY_SIZE(fds), 0), 0); +#else + cl_skip(); +#endif +} + +void test_poll__socket_cleanup(void) +{ +#ifdef GIT_WINDOWS_NATIVE + struct pollfd fds[POLL_MAX_DESCRIPTORS + 2] = { 0 }; + WSANETWORKEVENTS events; + SOCKET socket_handle; + + create_socket_pair(); + socket_handle = (SOCKET)_get_osfhandle(poll_test.sockets[0]); + poll_test.event = WSACreateEvent(); + cl_assert(poll_test.event != WSA_INVALID_EVENT); + cl_assert_equal_i(WSAEventSelect(socket_handle, poll_test.event, + FD_READ), 0); + poll_test.event_selected = 1; + + fds[0].fd = poll_test.sockets[0]; + open_pipes(fds + 1, POLL_MAX_DESCRIPTORS + 1); + + errno = 0; + cl_assert_equal_i(poll(fds, ARRAY_SIZE(fds), 0), -1); + cl_assert_equal_i(errno, EINVAL); + + cl_assert_equal_i(send( + (SOCKET)_get_osfhandle(poll_test.sockets[1]), + "x", 1, 0), 1); + cl_assert_equal_i(WaitForSingleObject(poll_test.event, 1000), + WAIT_OBJECT_0); + cl_assert_equal_i(WSAEnumNetworkEvents(socket_handle, poll_test.event, + &events), 0); +#else + cl_skip(); +#endif +} From 2a246fb6b29826890eb7d127b23e7b9e4d7d7b31 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 11 Dec 2018 12:17:49 +0100 Subject: [PATCH 195/229] fscache: implement an FSCache-aware is_mount_point() When FSCache is active, we can cache the reparse tag and use it directly to determine whether a path refers to an NTFS junction, without any additional, costly I/O. Note: this change only makes a difference with the next commit, which will make use of the FSCache in `git clean` (contingent on `core.fscache` set, of course). Signed-off-by: Johannes Schindelin --- compat/mingw.c | 2 ++ compat/mingw.h | 3 ++- compat/win32/fscache.c | 40 ++++++++++++++++++++++++++++++++++++++++ compat/win32/fscache.h | 1 + 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 8d667655244372..69896770fc7a8f 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -3484,6 +3484,8 @@ pid_t waitpid(pid_t pid, int *status, int options) return -1; } +int (*win32_is_mount_point)(struct strbuf *path) = mingw_is_mount_point; + int mingw_is_mount_point(struct strbuf *path) { WIN32_FIND_DATAW findbuf = { 0 }; diff --git a/compat/mingw.h b/compat/mingw.h index bae25483cf7ce0..4bbc6c5ce78759 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -40,7 +40,8 @@ static inline void convert_slashes(char *path) } struct strbuf; int mingw_is_mount_point(struct strbuf *path); -#define is_mount_point mingw_is_mount_point +extern int (*win32_is_mount_point)(struct strbuf *path); +#define is_mount_point win32_is_mount_point #define CAN_UNLINK_MOUNT_POINTS 1 #define PATH_SEP ';' char *mingw_query_user_email(void); diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index e247e9a6663131..26ae9ab1c1a464 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -515,6 +515,7 @@ int fscache_enable(size_t initial_size) /* redirect opendir and lstat to the fscache implementations */ opendir = fscache_opendir; lstat = fscache_lstat; + win32_is_mount_point = fscache_is_mount_point; } initialized++; LeaveCriticalSection(&fscache_cs); @@ -575,6 +576,7 @@ void fscache_disable(void) /* reset opendir and lstat to the original implementations */ opendir = dirent_opendir; lstat = mingw_lstat; + win32_is_mount_point = mingw_is_mount_point; } LeaveCriticalSection(&fscache_cs); @@ -662,6 +664,44 @@ int fscache_lstat(const char *filename, struct stat *st) return 0; } +/* + * is_mount_point() replacement, uses cache if enabled, otherwise falls + * back to mingw_is_mount_point(). + */ +int fscache_is_mount_point(struct strbuf *path) +{ + int dirlen, base, len; +#pragma GCC diagnostic push +#ifdef __clang__ +#pragma GCC diagnostic ignored "-Wflexible-array-extensions" +#endif + struct heap_fsentry key[2]; +#pragma GCC diagnostic pop + struct fsentry *fse; + struct fscache *cache = fscache_getcache(); + + if (!cache || !do_fscache_enabled(cache, path->buf)) + return mingw_is_mount_point(path); + + cache->lstat_requests++; + /* split path into path + name */ + len = path->len; + if (len && is_dir_sep(path->buf[len - 1])) + len--; + base = len; + while (base && !is_dir_sep(path->buf[base - 1])) + base--; + dirlen = base ? base - 1 : 0; + + /* lookup entry for path + name in cache */ + fsentry_init(&key[0].u.ent, NULL, path->buf, dirlen); + fsentry_init(&key[1].u.ent, &key[0].u.ent, path->buf + base, len - base); + fse = fscache_get(cache, &key[1].u.ent); + if (!fse) + return mingw_is_mount_point(path); + return fse->reparse_tag == IO_REPARSE_TAG_MOUNT_POINT; +} + typedef struct fscache_DIR { struct DIR base_dir; /* extend base struct DIR */ struct fsentry *pfsentry; diff --git a/compat/win32/fscache.h b/compat/win32/fscache.h index 042b247a542554..386c770a85d321 100644 --- a/compat/win32/fscache.h +++ b/compat/win32/fscache.h @@ -22,6 +22,7 @@ void fscache_flush(void); DIR *fscache_opendir(const char *dir); int fscache_lstat(const char *file_name, struct stat *buf); +int fscache_is_mount_point(struct strbuf *path); /* opaque fscache structure */ struct fscache; From b88639f6a888b16cb8b716d0c25901eec89bed8c Mon Sep 17 00:00:00 2001 From: Ben Peart Date: Thu, 15 Nov 2018 14:15:40 -0500 Subject: [PATCH 196/229] fscache: teach fscache to use NtQueryDirectoryFile Using FindFirstFileExW() requires the OS to allocate a 64K buffer for each directory and then free it when we call FindClose(). Update fscache to call the underlying kernel API NtQueryDirectoryFile so that we can do the buffer management ourselves. That allows us to allocate a single buffer for the lifetime of the cache and reuse it for each directory. This change improves performance of 'git status' by 18% in a repo with ~200K files and 30k folders. Documentation for NtQueryDirectoryFile can be found at: https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/ntifs/nf-ntifs-ntquerydirectoryfile https://docs.microsoft.com/en-us/windows/desktop/FileIO/file-attribute-constants https://docs.microsoft.com/en-us/windows/desktop/fileio/reparse-point-tags To determine if the specified directory is a symbolic link, inspect the FileAttributes member to see if the FILE_ATTRIBUTE_REPARSE_POINT flag is set. If so, EaSize will contain the reparse tag (this is a so far undocumented feature, but confirmed by the NTFS developers). To determine if the reparse point is a symbolic link (and not some other form of reparse point), test whether the tag value equals the value IO_REPARSE_TAG_SYMLINK. The NtQueryDirectoryFile() call works best (and on Windows 8.1 and earlier, it works *only*) with buffer sizes up to 64kB. Which is 32k wide characters, so let's use that as our buffer size. Signed-off-by: Ben Peart Signed-off-by: Johannes Schindelin --- compat/win32/fscache.c | 126 +++++++++++++++++++++++++++++---------- compat/win32/ntifs.h | 131 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 31 deletions(-) create mode 100644 compat/win32/ntifs.h diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 335c46dd36c544..973ae7efb246c9 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -7,6 +7,7 @@ #include "../../trace.h" #include "config.h" #include "../../mem-pool.h" +#include "ntifs.h" static volatile long initialized; static DWORD dwTlsIndex; @@ -26,6 +27,13 @@ struct fscache { unsigned int opendir_requests; unsigned int fscache_requests; unsigned int fscache_misses; + /* + * 32k wide characters translates to 64kB, which is the maximum that + * Windows 8.1 and earlier can handle. On network drives, not only + * the client's Windows version matters, but also the server's, + * therefore we need to keep this to 64kB. + */ + WCHAR buffer[32 * 1024]; }; static struct trace_key trace_fscache = TRACE_KEY_INIT(FSCACHE); @@ -166,29 +174,47 @@ static void fsentry_release(struct fsentry *fse) InterlockedDecrement(&(fse->u.refcnt)); } +static int xwcstoutfn(char *utf, int utflen, const wchar_t *wcs, int wcslen) +{ + if (!wcs || !utf || utflen < 1) { + errno = EINVAL; + return -1; + } + utflen = WideCharToMultiByte(CP_UTF8, 0, wcs, wcslen, utf, utflen, NULL, NULL); + if (utflen) + return utflen; + errno = ERANGE; + return -1; +} + /* - * Allocate and initialize an fsentry from a WIN32_FIND_DATA structure. + * Allocate and initialize an fsentry from a FILE_FULL_DIR_INFORMATION structure. */ static struct fsentry *fseentry_create_entry(struct fscache *cache, struct fsentry *list, - const WIN32_FIND_DATAW *fdata) + PFILE_FULL_DIR_INFORMATION fdata) { char buf[MAX_PATH * 3]; int len; struct fsentry *fse; - len = xwcstoutf(buf, fdata->cFileName, ARRAY_SIZE(buf)); + + len = xwcstoutfn(buf, ARRAY_SIZE(buf), fdata->FileName, fdata->FileNameLength / sizeof(wchar_t)); fse = fsentry_alloc(cache, list, buf, len); - fse->st_mode = file_attr_to_st_mode(fdata->dwFileAttributes, - IO_REPARSE_TAG_SYMLINK); + fse->st_mode = file_attr_to_st_mode(fdata->FileAttributes, + fdata->EaSize); fse->dirent.d_type = S_ISREG(fse->st_mode) ? DT_REG : S_ISDIR(fse->st_mode) ? DT_DIR : DT_LNK; - fse->u.s.st_size = (((off64_t) (fdata->nFileSizeHigh)) << 32) - | fdata->nFileSizeLow; - filetime_to_timespec(&(fdata->ftLastAccessTime), &(fse->u.s.st_atim)); - filetime_to_timespec(&(fdata->ftLastWriteTime), &(fse->u.s.st_mtim)); - filetime_to_timespec(&(fdata->ftCreationTime), &(fse->u.s.st_ctim)); + fse->u.s.st_size = S_ISLNK(fse->st_mode) ? MAX_PATH : + fdata->EndOfFile.LowPart | + (((off_t)fdata->EndOfFile.HighPart) << 32); + filetime_to_timespec((FILETIME *)&(fdata->LastAccessTime), + &(fse->u.s.st_atim)); + filetime_to_timespec((FILETIME *)&(fdata->LastWriteTime), + &(fse->u.s.st_mtim)); + filetime_to_timespec((FILETIME *)&(fdata->CreationTime), + &(fse->u.s.st_ctim)); return fse; } @@ -201,8 +227,10 @@ static struct fsentry *fseentry_create_entry(struct fscache *cache, static struct fsentry *fsentry_create_list(struct fscache *cache, const struct fsentry *dir, int *dir_not_found) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ - WIN32_FIND_DATAW fdata; + wchar_t pattern[MAX_PATH]; + NTSTATUS status; + IO_STATUS_BLOCK iosb; + PFILE_FULL_DIR_INFORMATION di; HANDLE h; int wlen; struct fsentry *list, **phead; @@ -218,15 +246,18 @@ static struct fsentry *fsentry_create_list(struct fscache *cache, const struct f return NULL; } - /* append optional '/' and wildcard '*' */ - if (wlen) - pattern[wlen++] = '/'; - pattern[wlen++] = '*'; - pattern[wlen] = 0; + /* handle CWD */ + if (!wlen) { + wlen = GetCurrentDirectoryW(ARRAY_SIZE(pattern), pattern); + if (!wlen || wlen >= (ssize_t)ARRAY_SIZE(pattern)) { + errno = wlen ? ENAMETOOLONG : err_win_to_posix(GetLastError()); + return NULL; + } + } - /* open find handle */ - h = FindFirstFileExW(pattern, FindExInfoBasic, &fdata, FindExSearchNameMatch, - NULL, FIND_FIRST_EX_LARGE_FETCH); + h = CreateFileW(pattern, FILE_LIST_DIRECTORY, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, NULL); if (h == INVALID_HANDLE_VALUE) { err = GetLastError(); *dir_not_found = 1; /* or empty directory */ @@ -243,22 +274,55 @@ static struct fsentry *fsentry_create_list(struct fscache *cache, const struct f /* walk directory and build linked list of fsentry structures */ phead = &list->next; - do { - *phead = fseentry_create_entry(cache, list, &fdata); + status = NtQueryDirectoryFile(h, NULL, 0, 0, &iosb, cache->buffer, + sizeof(cache->buffer), FileFullDirectoryInformation, FALSE, NULL, FALSE); + if (!NT_SUCCESS(status)) { + /* + * NtQueryDirectoryFile returns STATUS_INVALID_PARAMETER when + * asked to enumerate an invalid directory (ie it is a file + * instead of a directory). Verify that is the actual cause + * of the error. + */ + if (status == (NTSTATUS)STATUS_INVALID_PARAMETER) { + DWORD attributes = GetFileAttributesW(pattern); + if (!(attributes & FILE_ATTRIBUTE_DIRECTORY)) + status = ERROR_DIRECTORY; + } + goto Error; + } + di = (PFILE_FULL_DIR_INFORMATION)(cache->buffer); + for (;;) { + + *phead = fseentry_create_entry(cache, list, di); phead = &(*phead)->next; - } while (FindNextFileW(h, &fdata)); - /* remember result of last FindNextFile, then close find handle */ - err = GetLastError(); - FindClose(h); + /* If there is no offset in the entry, the buffer has been exhausted. */ + if (di->NextEntryOffset == 0) { + status = NtQueryDirectoryFile(h, NULL, 0, 0, &iosb, cache->buffer, + sizeof(cache->buffer), FileFullDirectoryInformation, FALSE, NULL, FALSE); + if (!NT_SUCCESS(status)) { + if (status == STATUS_NO_MORE_FILES) + break; + goto Error; + } + + di = (PFILE_FULL_DIR_INFORMATION)(cache->buffer); + continue; + } + + /* Advance to the next entry. */ + di = (PFILE_FULL_DIR_INFORMATION)(((PUCHAR)di) + di->NextEntryOffset); + } - /* return the list if we've got all the files */ - if (err == ERROR_NO_MORE_FILES) - return list; + CloseHandle(h); + return list; - /* otherwise release the list and return error */ +Error: + trace_printf_key(&trace_fscache, + "fscache: status(%ld) unable to query directory " + "contents '%s'\n", status, dir->dirent.d_name); + CloseHandle(h); fsentry_release(list); - errno = err_win_to_posix(err); return NULL; } diff --git a/compat/win32/ntifs.h b/compat/win32/ntifs.h new file mode 100644 index 00000000000000..64ed792c52f352 --- /dev/null +++ b/compat/win32/ntifs.h @@ -0,0 +1,131 @@ +#ifndef _NTIFS_ +#define _NTIFS_ + +/* + * Copy necessary structures and definitions out of the Windows DDK + * to enable calling NtQueryDirectoryFile() + */ + +typedef _Return_type_success_(return >= 0) LONG NTSTATUS; +#define NT_SUCCESS(Status) (((NTSTATUS)(Status)) >= 0) + +#if !defined(_NTSECAPI_) && !defined(_WINTERNL_) && \ + !defined(__UNICODE_STRING_DEFINED) +#define __UNICODE_STRING_DEFINED +typedef struct _UNICODE_STRING { + USHORT Length; + USHORT MaximumLength; + PWSTR Buffer; +} UNICODE_STRING; +typedef UNICODE_STRING *PUNICODE_STRING; +typedef const UNICODE_STRING *PCUNICODE_STRING; +#endif /* !_NTSECAPI_ && !_WINTERNL_ && !__UNICODE_STRING_DEFINED */ + +typedef enum _FILE_INFORMATION_CLASS { + FileDirectoryInformation = 1, + FileFullDirectoryInformation, + FileBothDirectoryInformation, + FileBasicInformation, + FileStandardInformation, + FileInternalInformation, + FileEaInformation, + FileAccessInformation, + FileNameInformation, + FileRenameInformation, + FileLinkInformation, + FileNamesInformation, + FileDispositionInformation, + FilePositionInformation, + FileFullEaInformation, + FileModeInformation, + FileAlignmentInformation, + FileAllInformation, + FileAllocationInformation, + FileEndOfFileInformation, + FileAlternateNameInformation, + FileStreamInformation, + FilePipeInformation, + FilePipeLocalInformation, + FilePipeRemoteInformation, + FileMailslotQueryInformation, + FileMailslotSetInformation, + FileCompressionInformation, + FileObjectIdInformation, + FileCompletionInformation, + FileMoveClusterInformation, + FileQuotaInformation, + FileReparsePointInformation, + FileNetworkOpenInformation, + FileAttributeTagInformation, + FileTrackingInformation, + FileIdBothDirectoryInformation, + FileIdFullDirectoryInformation, + FileValidDataLengthInformation, + FileShortNameInformation, + FileIoCompletionNotificationInformation, + FileIoStatusBlockRangeInformation, + FileIoPriorityHintInformation, + FileSfioReserveInformation, + FileSfioVolumeInformation, + FileHardLinkInformation, + FileProcessIdsUsingFileInformation, + FileNormalizedNameInformation, + FileNetworkPhysicalNameInformation, + FileIdGlobalTxDirectoryInformation, + FileIsRemoteDeviceInformation, + FileAttributeCacheInformation, + FileNumaNodeInformation, + FileStandardLinkInformation, + FileRemoteProtocolInformation, + FileMaximumInformation +} FILE_INFORMATION_CLASS, *PFILE_INFORMATION_CLASS; + +typedef struct _FILE_FULL_DIR_INFORMATION { + ULONG NextEntryOffset; + ULONG FileIndex; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER EndOfFile; + LARGE_INTEGER AllocationSize; + ULONG FileAttributes; + ULONG FileNameLength; + ULONG EaSize; + WCHAR FileName[1]; +} FILE_FULL_DIR_INFORMATION, *PFILE_FULL_DIR_INFORMATION; + +typedef struct _IO_STATUS_BLOCK { + union { + NTSTATUS Status; + PVOID Pointer; + } u; + ULONG_PTR Information; +} IO_STATUS_BLOCK, *PIO_STATUS_BLOCK; + +typedef VOID +(NTAPI *PIO_APC_ROUTINE)( + IN PVOID ApcContext, + IN PIO_STATUS_BLOCK IoStatusBlock, + IN ULONG Reserved); + +NTSYSCALLAPI +NTSTATUS +NTAPI +NtQueryDirectoryFile( + _In_ HANDLE FileHandle, + _In_opt_ HANDLE Event, + _In_opt_ PIO_APC_ROUTINE ApcRoutine, + _In_opt_ PVOID ApcContext, + _Out_ PIO_STATUS_BLOCK IoStatusBlock, + _Out_writes_bytes_(Length) PVOID FileInformation, + _In_ ULONG Length, + _In_ FILE_INFORMATION_CLASS FileInformationClass, + _In_ BOOLEAN ReturnSingleEntry, + _In_opt_ PUNICODE_STRING FileName, + _In_ BOOLEAN RestartScan +); + +#define STATUS_NO_MORE_FILES ((NTSTATUS)0x80000006L) + +#endif From 9bfe092cd1b84605a5cdfd1ba8cd5853609048a5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 19 Nov 2018 20:34:13 +0100 Subject: [PATCH 197/229] tests: use the correct path separator with BusyBox BusyBox-w32 is a true Win32 application, i.e. it does not come with a POSIX emulation layer. That also means that it does *not* use the Unix convention of separating the entries in the PATH variable using colons, but semicolons. However, there are also BusyBox ports to Windows which use a POSIX emulation layer such as Cygwin's or MSYS2's runtime, i.e. using colons as PATH separators. As a tell-tale, let's use the presence of semicolons in the PATH variable: on Unix, it is highly unlikely that it contains semicolons, and on Windows (without POSIX emulation), it is virtually guaranteed, as everybody should have both $SYSTEMROOT and $SYSTEMROOT/system32 in their PATH. Signed-off-by: Johannes Schindelin --- t/interop/interop-lib.sh | 8 ++++++-- t/lib-proto-disable.sh | 2 +- t/t0021-conversion.sh | 2 +- t/t0060-path-utils.sh | 24 ++++++++++++------------ t/t0061-run-command.sh | 6 +++--- t/t0300-credentials.sh | 2 +- t/t1504-ceiling-dirs.sh | 10 +++++----- t/t2300-cd-to-toplevel.sh | 2 +- t/t3418-rebase-continue.sh | 4 ++-- t/t5615-alternate-env.sh | 4 ++-- t/t5802-connect-helper.sh | 2 +- t/t7006-pager.sh | 4 ++-- t/t7606-merge-custom.sh | 2 +- t/t7811-grep-open.sh | 2 +- t/t9003-help-autocorrect.sh | 2 +- t/t9800-git-p4-basic.sh | 2 +- t/test-lib.sh | 17 +++++++++++++---- 17 files changed, 54 insertions(+), 41 deletions(-) diff --git a/t/interop/interop-lib.sh b/t/interop/interop-lib.sh index 1b5864d2a7f22c..1facc69d97741a 100644 --- a/t/interop/interop-lib.sh +++ b/t/interop/interop-lib.sh @@ -4,6 +4,10 @@ . ../../GIT-BUILD-OPTIONS INTEROP_ROOT=$(pwd) BUILD_ROOT=$INTEROP_ROOT/build +case "$PATH" in +*\;*) PATH_SEP=\; ;; +*) PATH_SEP=: ;; +esac build_version () { if test -z "$1" @@ -57,7 +61,7 @@ wrap_git () { write_script "$1" <<-EOF GIT_EXEC_PATH="$2" export GIT_EXEC_PATH - PATH="$2:\$PATH" + PATH="$2$PATH_SEP\$PATH" export GIT_EXEC_PATH exec git "\$@" EOF @@ -71,7 +75,7 @@ generate_wrappers () { echo >&2 fatal: test tried to run generic git: $* exit 1 EOF - PATH=$(pwd)/.bin:$PATH + PATH=$(pwd)/.bin$PATH_SEP$PATH } VERSION_A=${GIT_TEST_VERSION_A:-$VERSION_A} diff --git a/t/lib-proto-disable.sh b/t/lib-proto-disable.sh index 890622be81642b..9db481e1be15b2 100644 --- a/t/lib-proto-disable.sh +++ b/t/lib-proto-disable.sh @@ -214,7 +214,7 @@ setup_ext_wrapper () { cd "$TRASH_DIRECTORY/remote" && eval "$*" EOF - PATH=$TRASH_DIRECTORY:$PATH && + PATH=$TRASH_DIRECTORY$PATH_SEP$PATH && export TRASH_DIRECTORY ' } diff --git a/t/t0021-conversion.sh b/t/t0021-conversion.sh index 632915086c521d..a0f093649df625 100755 --- a/t/t0021-conversion.sh +++ b/t/t0021-conversion.sh @@ -8,7 +8,7 @@ export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME . ./test-lib.sh . "$TEST_DIRECTORY"/lib-terminal.sh -PATH=$PWD:$PATH +PATH=$PWD$PATH_SEP$PATH TEST_ROOT="$(pwd)" write_script <<\EOF "$TEST_ROOT/rot13.sh" diff --git a/t/t0060-path-utils.sh b/t/t0060-path-utils.sh index 3cdc4738644dbc..5abfa202c19dca 100755 --- a/t/t0060-path-utils.sh +++ b/t/t0060-path-utils.sh @@ -147,25 +147,25 @@ ancestor /foo /fo -1 ancestor /foo /foo -1 ancestor /foo /bar -1 ancestor /foo /foo/bar -1 -ancestor /foo /foo:/bar -1 -ancestor /foo /:/foo:/bar 0 -ancestor /foo /foo:/:/bar 0 -ancestor /foo /:/bar:/foo 0 +ancestor /foo "/foo$PATH_SEP/bar" -1 +ancestor /foo "/$PATH_SEP/foo$PATH_SEP/bar" 0 +ancestor /foo "/foo$PATH_SEP/$PATH_SEP/bar" 0 +ancestor /foo "/$PATH_SEP/bar$PATH_SEP/foo" 0 ancestor /foo/bar / 0 ancestor /foo/bar /fo -1 ancestor /foo/bar /foo 4 ancestor /foo/bar /foo/ba -1 -ancestor /foo/bar /:/fo 0 -ancestor /foo/bar /foo:/foo/ba 4 +ancestor /foo/bar "/$PATH_SEP/fo" 0 +ancestor /foo/bar "/foo$PATH_SEP/foo/ba" 4 ancestor /foo/bar /bar -1 ancestor /foo/bar /fo -1 -ancestor /foo/bar /foo:/bar 4 -ancestor /foo/bar /:/foo:/bar 4 -ancestor /foo/bar /foo:/:/bar 4 -ancestor /foo/bar /:/bar:/fo 0 -ancestor /foo/bar /:/bar 0 +ancestor /foo/bar "/foo$PATH_SEP/bar" 4 +ancestor /foo/bar "/$PATH_SEP/foo$PATH_SEP/bar" 4 +ancestor /foo/bar "/foo$PATH_SEP/$PATH_SEP/bar" 4 +ancestor /foo/bar "/$PATH_SEP/bar$PATH_SEP/fo" 0 +ancestor /foo/bar "/$PATH_SEP/bar" 0 ancestor /foo/bar /foo 4 -ancestor /foo/bar /foo:/bar 4 +ancestor /foo/bar "/foo$PATH_SEP/bar" 4 ancestor /foo/bar /bar -1 # Windows-specific: DOS drives, network shares diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh index c74deaee1ac8fb..967d3f39fe74af 100755 --- a/t/t0061-run-command.sh +++ b/t/t0061-run-command.sh @@ -69,7 +69,7 @@ test_expect_success 'run_command does not try to execute a directory' ' cat bin2/greet EOF - PATH=$PWD/bin1:$PWD/bin2:$PATH \ + PATH=$PWD/bin1$PATH_SEP$PWD/bin2$PATH_SEP$PATH \ test-tool run-command run-command greet >actual 2>err && test_cmp bin2/greet actual && test_must_be_empty err @@ -86,7 +86,7 @@ test_expect_success POSIXPERM 'run_command passes over non-executable file' ' cat bin2/greet EOF - PATH=$PWD/bin1:$PWD/bin2:$PATH \ + PATH=$PWD/bin1$PATH_SEP$PWD/bin2$PATH_SEP$PATH \ test-tool run-command run-command greet >actual 2>err && test_cmp bin2/greet actual && test_must_be_empty err @@ -106,7 +106,7 @@ test_expect_success POSIXPERM,SANITY 'unreadable directory in PATH' ' git config alias.nitfol "!echo frotz" && chmod a-rx local-command && ( - PATH=./local-command:$PATH && + PATH=./local-command$PATH_SEP$PATH && git nitfol >actual ) && echo frotz >expect && diff --git a/t/t0300-credentials.sh b/t/t0300-credentials.sh index ea11cdb4dcd251..e8774485b82c28 100755 --- a/t/t0300-credentials.sh +++ b/t/t0300-credentials.sh @@ -80,7 +80,7 @@ test_expect_success 'setup helper scripts' ' printf "username=\\007latrix Lestrange\\n" EOF - PATH="$PWD:$PATH" + PATH="$PWD$PATH_SEP$PATH" ' test_expect_success 'credential_fill invokes helper' ' diff --git a/t/t1504-ceiling-dirs.sh b/t/t1504-ceiling-dirs.sh index e04420f4368b93..ff9fb804827b59 100755 --- a/t/t1504-ceiling-dirs.sh +++ b/t/t1504-ceiling-dirs.sh @@ -84,9 +84,9 @@ then GIT_CEILING_DIRECTORIES="$TRASH_ROOT/top/" test_fail subdir_ceil_at_top_slash - GIT_CEILING_DIRECTORIES=":$TRASH_ROOT/top" + GIT_CEILING_DIRECTORIES="$PATH_SEP$TRASH_ROOT/top" test_prefix subdir_ceil_at_top_no_resolve "sub/dir/" - GIT_CEILING_DIRECTORIES=":$TRASH_ROOT/top/" + GIT_CEILING_DIRECTORIES="$PATH_SEP$TRASH_ROOT/top/" test_prefix subdir_ceil_at_top_slash_no_resolve "sub/dir/" fi @@ -116,13 +116,13 @@ GIT_CEILING_DIRECTORIES="$TRASH_ROOT/subdi" test_prefix subdir_ceil_at_subdi_slash "sub/dir/" -GIT_CEILING_DIRECTORIES="/foo:$TRASH_ROOT/sub" +GIT_CEILING_DIRECTORIES="/foo$PATH_SEP$TRASH_ROOT/sub" test_fail second_of_two -GIT_CEILING_DIRECTORIES="$TRASH_ROOT/sub:/bar" +GIT_CEILING_DIRECTORIES="$TRASH_ROOT/sub$PATH_SEP/bar" test_fail first_of_two -GIT_CEILING_DIRECTORIES="/foo:$TRASH_ROOT/sub:/bar" +GIT_CEILING_DIRECTORIES="/foo$PATH_SEP$TRASH_ROOT/sub$PATH_SEP/bar" test_fail second_of_three diff --git a/t/t2300-cd-to-toplevel.sh b/t/t2300-cd-to-toplevel.sh index c8de6d8a190220..91f523d5198d8d 100755 --- a/t/t2300-cd-to-toplevel.sh +++ b/t/t2300-cd-to-toplevel.sh @@ -16,7 +16,7 @@ test_cd_to_toplevel () { test_expect_success $3 "$2" ' ( cd '"'$1'"' && - PATH="$EXEC_PATH:$PATH" && + PATH="$EXEC_PATH$PATH_SEP$PATH" && . git-sh-setup && cd_to_toplevel && [ "$(pwd -P)" = "$TOPLEVEL" ] diff --git a/t/t3418-rebase-continue.sh b/t/t3418-rebase-continue.sh index 8c43ef64fce3d4..d6d50b481bd698 100755 --- a/t/t3418-rebase-continue.sh +++ b/t/t3418-rebase-continue.sh @@ -82,7 +82,7 @@ test_expect_success 'rebase --continue remembers merge strategy and options' ' rm -f actual && ( - PATH=./test-bin:$PATH && + PATH=./test-bin$PATH_SEP$PATH && test_must_fail git rebase -s funny -X"option=arg with space" \ -Xop\"tion\\ -X"new${LF}line " main topic ) && @@ -91,7 +91,7 @@ test_expect_success 'rebase --continue remembers merge strategy and options' ' echo "Resolved" >F2 && git add F2 && ( - PATH=./test-bin:$PATH && + PATH=./test-bin$PATH_SEP$PATH && git rebase --continue ) && test_cmp expect actual diff --git a/t/t5615-alternate-env.sh b/t/t5615-alternate-env.sh index 9d6aa2187f2aaa..1bfeccdeb49958 100755 --- a/t/t5615-alternate-env.sh +++ b/t/t5615-alternate-env.sh @@ -39,7 +39,7 @@ test_expect_success 'access alternate via absolute path' ' ' test_expect_success 'access multiple alternates' ' - check_obj "$PWD/one.git/objects:$PWD/two.git/objects" <<-EOF + check_obj "$PWD/one.git/objects$PATH_SEP$PWD/two.git/objects" <<-EOF $one blob $two blob EOF @@ -75,7 +75,7 @@ test_expect_success 'access alternate via relative path (subdir)' ' quoted='"one.git\057objects"' unquoted='two.git/objects' test_expect_success 'mix of quoted and unquoted alternates' ' - check_obj "$quoted:$unquoted" <<-EOF + check_obj "$quoted$PATH_SEP$unquoted" <<-EOF $one blob $two blob EOF diff --git a/t/t5802-connect-helper.sh b/t/t5802-connect-helper.sh index a7be375bceb8d3..26cbcebf3b2b24 100755 --- a/t/t5802-connect-helper.sh +++ b/t/t5802-connect-helper.sh @@ -86,7 +86,7 @@ test_expect_success 'set up fake git-daemon' ' "$TRASH_DIRECTORY/remote" EOF export TRASH_DIRECTORY && - PATH=$TRASH_DIRECTORY:$PATH + PATH=$TRASH_DIRECTORY$PATH_SEP$PATH ' test_expect_success 'ext command can connect to git daemon (no vhost)' ' diff --git a/t/t7006-pager.sh b/t/t7006-pager.sh index 75ca7a25daed91..cb6def62e65424 100755 --- a/t/t7006-pager.sh +++ b/t/t7006-pager.sh @@ -54,7 +54,7 @@ test_expect_success !MINGW,TTY 'LESS and LV envvars set by git-sh-setup' ' sane_unset LESS LV && PAGER="env >pager-env.out; wc" && export PAGER && - PATH="$(git --exec-path):$PATH" && + PATH="$(git --exec-path)$PATH_SEP$PATH" && export PATH && test_terminal sh -c ". git-sh-setup && git_pager" ) && @@ -388,7 +388,7 @@ test_default_pager() { EOF chmod +x \$less && ( - PATH=.:\$PATH && + PATH=.$PATH_SEP\$PATH && export PATH && $full_command ) && diff --git a/t/t7606-merge-custom.sh b/t/t7606-merge-custom.sh index 0ae2f59b1d8f86..2c306779fe6fd7 100755 --- a/t/t7606-merge-custom.sh +++ b/t/t7606-merge-custom.sh @@ -23,7 +23,7 @@ test_expect_success 'set up custom strategy' ' EOF chmod +x git-merge-theirs && - PATH=.:$PATH && + PATH=.$PATH_SEP$PATH && export PATH ' diff --git a/t/t7811-grep-open.sh b/t/t7811-grep-open.sh index 3160be59fd2e26..1a98d733dceb86 100755 --- a/t/t7811-grep-open.sh +++ b/t/t7811-grep-open.sh @@ -52,7 +52,7 @@ test_expect_success SIMPLEPAGER 'git grep -O' ' EOF echo grep.h >expect.notless && - PATH=.:$PATH git grep -O GREP_PATTERN >out && + PATH=.$PATH_SEP$PATH git grep -O GREP_PATTERN >out && { test_cmp expect.less pager-args || test_cmp expect.notless pager-args diff --git a/t/t9003-help-autocorrect.sh b/t/t9003-help-autocorrect.sh index 87b438fcac7bb9..a6f8aaebeb56d7 100755 --- a/t/t9003-help-autocorrect.sh +++ b/t/t9003-help-autocorrect.sh @@ -13,7 +13,7 @@ test_expect_success 'setup' ' echo distimdistim was called EOF - PATH="$PATH:." && + PATH="$PATH$PATH_SEP." && export PATH && git commit --allow-empty -m "a single log entry" && diff --git a/t/t9800-git-p4-basic.sh b/t/t9800-git-p4-basic.sh index 140a3a7f24b26a..f55f6758d5e8a8 100755 --- a/t/t9800-git-p4-basic.sh +++ b/t/t9800-git-p4-basic.sh @@ -286,7 +286,7 @@ test_expect_success 'exit when p4 fails to produce marshaled output' ' EOF chmod 755 badp4dir/p4 && ( - PATH="$TRASH_DIRECTORY/badp4dir:$PATH" && + PATH="$TRASH_DIRECTORY/badp4dir$PATH_SEP$PATH" && export PATH && test_expect_code 1 git p4 clone --dest="$git" //depot >errs 2>&1 ) && diff --git a/t/test-lib.sh b/t/test-lib.sh index 37af8a3037b086..1b4622838545dc 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -40,6 +40,15 @@ case "${GIT_TEST_USE_SET_E:-false}" in ;; esac +# On Unix/Linux, the path separator is the colon, on other systems it +# may be different, though. On Windows, for example, it is a semicolon. +# If the PATH variable contains semicolons, it is pretty safe to assume +# that the path separator is a semicolon. +case "$PATH" in +*\;*) PATH_SEP=\; ;; +*) PATH_SEP=: ;; +esac + # Test the binaries we have just built. The tests are kept in # t/ subdirectory and are run in 'trash directory' subdirectory. if test -z "$TEST_DIRECTORY" @@ -1423,7 +1432,7 @@ then done done IFS=$OLDIFS - PATH=$GIT_VALGRIND/bin:$PATH + PATH=$GIT_VALGRIND/bin$PATH_SEP$PATH GIT_EXEC_PATH=$GIT_VALGRIND/bin export GIT_VALGRIND GIT_VALGRIND_MODE="$valgrind" @@ -1435,7 +1444,7 @@ elif test -n "$GIT_TEST_INSTALLED" then GIT_EXEC_PATH=$($GIT_TEST_INSTALLED/git --exec-path) || error "Cannot run git from $GIT_TEST_INSTALLED." - PATH=$GIT_TEST_INSTALLED:$GIT_BUILD_DIR/t/helper:$PATH + PATH=$GIT_TEST_INSTALLED$PATH_SEP$GIT_BUILD_DIR/t/helper$PATH_SEP$PATH GIT_EXEC_PATH=${GIT_TEST_EXEC_PATH:-$GIT_EXEC_PATH} else # normal case, use ../bin-wrappers only unless $with_dashes: if test -n "$no_bin_wrappers" @@ -1451,12 +1460,12 @@ else # normal case, use ../bin-wrappers only unless $with_dashes: fi with_dashes=t fi - PATH="$git_bin_dir:$PATH" + PATH="$git_bin_dir$PATH_SEP$PATH" fi GIT_EXEC_PATH=$GIT_BUILD_DIR if test -n "$with_dashes" then - PATH="$GIT_BUILD_DIR:$GIT_BUILD_DIR/t/helper:$PATH" + PATH="$GIT_BUILD_DIR$PATH_SEP$GIT_BUILD_DIR/t/helper$PATH_SEP$PATH" fi fi GIT_TEMPLATE_DIR="$GIT_TEST_TEMPLATE_DIR" From fb3806d99375979d9d745a0b5249707e1ae14a87 Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Tue, 1 Sep 2026 11:38:35 -0700 Subject: [PATCH 198/229] parallel-checkout: limit worker count to what poll() can wait on On Windows, `git checkout` and `git reset --hard` can abort with *** stack smashing detected ***: terminated and exit code 0xC0000409 (STATUS_STACK_BUFFER_OVERRUN) when checkout.workers is large, or when it is set to 0 on a machine with many logical processors. gather_results_from_workers() polls one pipe per checkout worker. Windows has no native poll(), so compat/poll emulates it with MsgWaitForMultipleObjects(), which cannot wait on more than POLL_MAX_DESCRIPTORS descriptors at once. compat/poll collects one handle per polled descriptor in a fixed stack array, so a higher worker count writes past the end of that array and corrupts the stack. run_parallel_checkout() clamped num_workers only by the number of files. Clamp it to POLL_MAX_DESCRIPTORS as well. That function is the single choke point before the workers start and the poll() loop runs. Clamp silently: fewer workers is correct behaviour, and a warning would fire on every checkout on a large machine. A single poll() loop cannot usefully drive more readers than this anyway. On platforms with a native poll() the limit is INT_MAX, so the clamp is a no-op. The problem became reachable in 2.54. Before that, online_cpus() used GetSystemInfo(), which reports only the processors in the current processor group, and a group holds at most 64. That accidental ceiling kept the array in bounds. The move to GetLogicalProcessorInformationEx() is correct and reports the true system-wide count, which exposed the latent bug. Document the cap, because checkout.workers is otherwise described as using one worker per logical core with no upper bound. Add a test that asserts the clamp. test_checkout_workers() counts the workers that were actually spawned, so the test can check the effective count rather than only that the checkout succeeded. The test is limited to Windows, because that is the only platform where the cap applies. Signed-off-by: Tyrie Vella --- Documentation/config/checkout.adoc | 5 ++++ parallel-checkout.c | 7 ++++++ t/t2080-parallel-checkout-basics.sh | 36 +++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/Documentation/config/checkout.adoc b/Documentation/config/checkout.adoc index e35d21296978fe..45951bf38a5e3c 100644 --- a/Documentation/config/checkout.adoc +++ b/Documentation/config/checkout.adoc @@ -30,6 +30,11 @@ commands or functionality in the future. all commands that perform checkout. E.g. checkout, clone, reset, sparse-checkout, etc. + +On Windows the number of workers is capped at 62, because the `poll()` +emulation cannot wait on more worker pipes than that. A higher configured +value, including the logical core count on a machine with many cores, is +silently reduced to the cap. ++ NOTE: Parallel checkout usually delivers better performance for repositories located on SSDs or over NFS. For repositories on spinning disks and/or machines with a small number of cores, the default sequential checkout often performs diff --git a/parallel-checkout.c b/parallel-checkout.c index 1eb277a0fc0a55..4595cf4e8d9250 100644 --- a/parallel-checkout.c +++ b/parallel-checkout.c @@ -671,6 +671,13 @@ int run_parallel_checkout(struct checkout *state, int num_workers, int threshold if (parallel_checkout.nr < num_workers) num_workers = parallel_checkout.nr; + /* + * gather_results_from_workers() polls one pipe per worker, so the + * worker count must stay within what poll() can wait on. + */ + if (num_workers > POLL_MAX_DESCRIPTORS) + num_workers = POLL_MAX_DESCRIPTORS; + if (num_workers <= 1 || parallel_checkout.nr < threshold) { write_items_sequentially(state); } else { diff --git a/t/t2080-parallel-checkout-basics.sh b/t/t2080-parallel-checkout-basics.sh index 322d26708e0bfd..51d4e1db4d9cbb 100755 --- a/t/t2080-parallel-checkout-basics.sh +++ b/t/t2080-parallel-checkout-basics.sh @@ -319,5 +319,41 @@ test_expect_success MINGW 'parallel checkout with fscache does not fail on new d test_cmp expect2 sub/deep/dir/file2.txt ) ' +# Windows has no native poll(). compat/poll emulates it with +# MsgWaitForMultipleObjects(), which cannot wait on more than +# MAXIMUM_WAIT_OBJECTS objects, so run_parallel_checkout() caps the worker +# count at MAXIMUM_WAIT_OBJECTS - 2. Without that cap, compat/poll collected +# one wait handle per polled worker pipe in a fixed-size stack array and +# smashed the stack. +# +# MAXIMUM_WAIT_OBJECTS is 64, hence the expected 62 below. The test is +# MINGW-only because the cap only exists there; on other platforms the +# requested 200 workers are used as-is. +test_expect_success MINGW 'checkout caps workers at the poll limit' ' + test_when_finished "rm -rf many-workers" && + git init many-workers && + ( + cd many-workers && + mkdir dir && + for i in $(test_seq 1 200) + do + echo "content $i" >dir/file$i || return 1 + done && + git add -A && + git commit -q -m base && + + git checkout -q -b other && + for i in $(test_seq 1 200) + do + echo "changed $i" >dir/file$i || return 1 + done && + git commit -q -a -m changed && + git checkout -q - + ) && + + set_checkout_config 200 1 && + test_checkout_workers 62 git -C many-workers checkout other && + verify_checkout many-workers +' test_done From 99014a547a09895b0dc6dfabf59570c1d79a493d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 11 Dec 2018 12:17:49 +0100 Subject: [PATCH 199/229] clean: make use of FSCache The `git clean` command needs to enumerate plenty of files and directories, and can therefore benefit from the FSCache. Signed-off-by: Johannes Schindelin --- builtin/clean.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builtin/clean.c b/builtin/clean.c index 6ed555000f9a41..e15d595c3dc7cc 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -1042,6 +1042,7 @@ int cmd_clean(int argc, if (repo_read_index(the_repository) < 0) die(_("index file corrupt")); + enable_fscache(the_repository->index->cache_nr); pl = add_pattern_list(&dir, EXC_CMDL, "--exclude option"); for (i = 0; i < exclude_list.nr; i++) @@ -1116,6 +1117,7 @@ int cmd_clean(int argc, } } + disable_fscache(); strbuf_release(&abs_path); strbuf_release(&buf); string_list_clear(&del_list, 0); From e321f84b5c6e286c800f4400ea5a8871772729b3 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 30 Jun 2017 00:35:40 +0200 Subject: [PATCH 200/229] mingw: only use Bash-ism `builtin pwd -W` when available Traditionally, Git for Windows' SDK uses Bash as its default shell. However, other Unix shells are available, too. Most notably, the Win32 port of BusyBox comes with `ash` whose `pwd` command already prints Windows paths as Git for Windows wants them, while there is not even a `builtin` command. Therefore, let's be careful not to override `pwd` unless we know that the `builtin` command is available. Signed-off-by: Johannes Schindelin --- git-sh-setup.sh | 14 ++++++++++---- t/test-lib.sh | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/git-sh-setup.sh b/git-sh-setup.sh index fad4f9df94e143..c51ad34148ccf3 100644 --- a/git-sh-setup.sh +++ b/git-sh-setup.sh @@ -306,10 +306,16 @@ case $(uname -s) in /usr/bin/find "$@" } fi - # git sees Windows-style pwd - pwd () { - builtin pwd -W - } + # On Windows, Git wants Windows paths. But /usr/bin/pwd spits out + # Unix-style paths. At least in Bash, we have a builtin pwd that + # understands the -W option to force "mixed" paths, i.e. with drive + # prefix but still with forward slashes. Let's use that, if available. + if type builtin >/dev/null 2>&1 + then + pwd () { + builtin pwd -W + } + fi is_absolute_path () { case "$1" in [/\\]* | [A-Za-z]:*) diff --git a/t/test-lib.sh b/t/test-lib.sh index 1b4622838545dc..9ddb166d12f161 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1735,10 +1735,16 @@ Darwin) /usr/bin/find "$@" } fi - # git sees Windows-style pwd - pwd () { - builtin pwd -W - } + # On Windows, Git wants Windows paths. But /usr/bin/pwd spits out + # Unix-style paths. At least in Bash, we have a builtin pwd that + # understands the -W option to force "mixed" paths, i.e. with drive + # prefix but still with forward slashes. Let's use that, if available. + if type builtin >/dev/null 2>&1 + then + pwd () { + builtin pwd -W + } + fi # no POSIX permissions # backslashes in pathspec are converted to '/' # exec does not inherit the PID From cec60a285941db1a0bffa10cc96306e57ffb680d Mon Sep 17 00:00:00 2001 From: Tyrie Vella Date: Tue, 1 Sep 2026 11:38:36 -0700 Subject: [PATCH 201/229] run-command: limit concurrent children to what poll() can wait on pp_buffer_io() polls one pipe for each child that is sending output, and a second one for each child that is being fed on stdin. On Windows poll() is emulated with MsgWaitForMultipleObjects(), which cannot wait on more than POLL_MAX_DESCRIPTORS descriptors at once. A job count above that limit is reachable in practice. fetch.parallel, submodule.fetchJobs and hook.jobs all accept an explicit value, and a value of 0 means "use online_cpus()", which on a machine with many cores is well above the limit. Before the previous commit such a run corrupted the stack. Now poll() returns EINVAL, and pp_buffer_io() turns that into die_errno("poll"), so the operation fails outright. Limit how many children run at the same time, so that a large job count degrades into less concurrency instead of an error. Only concurrency is limited. The configured maximum is still used for the size of the child and pollfd arrays, and is still reported by the trace, so the number of tasks that are run in total does not change. Unused pollfd slots hold -1 and are skipped by poll(), so the larger array costs nothing. Divide the limit by two, because a child can hold two descriptors: one for its output and one for its input. Callers that group output are the only ones affected; with opts.ungroup set the caller does its own I/O and poll() is not involved. On platforms with a native poll() there is no such limit, POLL_MAX_DESCRIPTORS is INT_MAX, and this is a no-op. Signed-off-by: Tyrie Vella --- builtin/fetch.c | 1 + builtin/submodule--helper.c | 1 + hook.c | 1 + run-command.c | 20 ++++++++++- run-command.h | 6 ++++ submodule.c | 1 + t/helper/test-run-command.c | 25 ++++++++++++- t/t0061-run-command.sh | 71 +++++++++++++++++++++++++++++++++++++ 8 files changed, 124 insertions(+), 2 deletions(-) diff --git a/builtin/fetch.c b/builtin/fetch.c index db08197cabcfd8..c2bc8ab01f7fa4 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -2415,6 +2415,7 @@ static int fetch_multiple(struct string_list *list, int max_children, .tr2_label = "parallel/fetch", .processes = max_children, + .no_stdin_pipe = 1, .get_next_task = &fetch_next_remote, .start_failure = &fetch_failed_to_start, diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 49883af6ffc619..c8c6e174418e9d 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -2921,6 +2921,7 @@ static int update_submodules(struct update_data *update_data) .tr2_label = "parallel/update", .processes = update_data->max_jobs, + .no_stdin_pipe = 1, .get_next_task = update_clone_get_next_task, .start_failure = update_clone_start_failure, diff --git a/hook.c b/hook.c index d10eef4763c679..5bd0935bae307a 100644 --- a/hook.c +++ b/hook.c @@ -798,6 +798,7 @@ int run_hooks_opt(struct repository *r, const char *hook_name, .processes = jobs, .ungroup = jobs == 1, + .no_stdin_pipe = !options->feed_pipe, .get_next_task = pick_next_hook, .start_failure = notify_start_failure, diff --git a/run-command.c b/run-command.c index ce84db87824262..27c31af8b0bb41 100644 --- a/run-command.c +++ b/run-command.c @@ -1658,6 +1658,9 @@ static int pp_start_one(struct parallel_processes *pp, } return 1; } + if (opts->no_stdin_pipe && pp->children[i].process.in < 0) + BUG("get_next_task requested a stdin pipe despite " + "no_stdin_pipe"); if (!opts->ungroup) { pp->children[i].process.err = -1; pp->children[i].process.stdout_to_stderr = 1; @@ -1893,6 +1896,7 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts) int i, code; int timeout = 100; int spawn_cap = 4; + size_t max_live; struct parallel_processes_for_signal pp_sig; struct parallel_processes pp = { .buffered_output = STRBUF_INIT, @@ -1902,6 +1906,20 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts) const char *tr2_label = opts->tr2_label; const int do_trace2 = tr2_category && tr2_label; + /* + * Unless the caller handles its own output, pp_buffer_io() polls one + * output pipe per child and, unless excluded by no_stdin_pipe, may also + * poll an input pipe. Limit the number of live children so that all of + * their descriptors fit in one poll() call. + */ + max_live = opts->processes; + if (!opts->ungroup) { + size_t fds_per_process = opts->no_stdin_pipe ? 1 : 2; + + if (max_live > POLL_MAX_DESCRIPTORS / fds_per_process) + max_live = POLL_MAX_DESCRIPTORS / fds_per_process; + } + if (do_trace2) trace2_region_enter_printf(tr2_category, tr2_label, NULL, "max:%"PRIuMAX, @@ -1923,7 +1941,7 @@ void run_processes_parallel(const struct run_process_parallel_opts *opts) while (1) { for (i = 0; i < spawn_cap && !pp.shutdown && - pp.nr_processes < opts->processes; + pp.nr_processes < max_live; i++) { code = pp_start_one(&pp, opts); if (!code) diff --git a/run-command.h b/run-command.h index c2fad4f0f8380c..331c9391e8e2d5 100644 --- a/run-command.h +++ b/run-command.h @@ -488,6 +488,12 @@ struct run_process_parallel_opts */ unsigned int ungroup:1; + /** + * no_stdin_pipe: set if get_next_task will never request a pipe by + * setting child_process.in to -1. + */ + unsigned int no_stdin_pipe:1; + /** * get_next_task: See get_next_task_fn() above. This must be * specified. diff --git a/submodule.c b/submodule.c index 5c8dce4c344d65..48edad9eda34f6 100644 --- a/submodule.c +++ b/submodule.c @@ -1862,6 +1862,7 @@ int fetch_submodules(struct repository *r, .tr2_label = "parallel/fetch", .processes = max_parallel_jobs, + .no_stdin_pipe = 1, .get_next_task = get_next_submodule, .start_failure = fetch_start_failure, diff --git a/t/helper/test-run-command.c b/t/helper/test-run-command.c index 4a56456894ccff..f943c8214d4dec 100644 --- a/t/helper/test-run-command.c +++ b/t/helper/test-run-command.c @@ -20,13 +20,14 @@ #include "wildmatch.h" static int number_callbacks; +static int max_callbacks = 4; static int parallel_next(struct child_process *cp, struct strbuf *err, void *cb, void **task_cb) { struct child_process *d = cb; - if (number_callbacks >= 4) + if (number_callbacks >= max_callbacks) return 0; strvec_pushv(&cp->args, d->args.v); @@ -195,6 +196,7 @@ static int testsuite(int argc, const char **argv) OPT_END() }; struct run_process_parallel_opts opts = { + .no_stdin_pipe = 1, .get_next_task = next_test, .start_failure = test_failed, .feed_pipe = test_stdin_pipe_feed, @@ -442,6 +444,16 @@ static int inherit_handle_child(void) int cmd__run_command(int argc, const char **argv) { struct child_process proc = CHILD_PROCESS_INIT; + const char * const parallel_usage[] = { + "test-tool run-command [] " + " [...]", + NULL + }; + struct option parallel_options[] = { + OPT_INTEGER_F(0, "tasks", &max_callbacks, + "number of tasks to generate", PARSE_OPT_NONEG), + OPT_END() + }; int jobs; int ret; struct run_process_parallel_opts opts = { @@ -495,17 +507,28 @@ int cmd__run_command(int argc, const char **argv) opts.ungroup = 1; } + argc = parse_options(argc - 1, argv + 1, NULL, parallel_options, + parallel_usage, PARSE_OPT_STOP_AT_NON_OPTION | + PARSE_OPT_KEEP_ARGV0); + if (argc < 3) + usage_with_options(parallel_usage, parallel_options); + if (max_callbacks < 0) + die("--tasks cannot be negative"); + jobs = atoi(argv[2]); strvec_clear(&proc.args); strvec_pushv(&proc.args, (const char **)argv + 3); if (!strcmp(argv[1], "run-command-parallel")) { + opts.no_stdin_pipe = 1; opts.get_next_task = parallel_next; opts.task_finished = task_finished_quiet; } else if (!strcmp(argv[1], "run-command-abort")) { + opts.no_stdin_pipe = 1; opts.get_next_task = parallel_next; opts.task_finished = task_finished; } else if (!strcmp(argv[1], "run-command-no-jobs")) { + opts.no_stdin_pipe = 1; opts.get_next_task = no_job; opts.task_finished = task_finished; } else if (!strcmp(argv[1], "run-command-stdin")) { diff --git a/t/t0061-run-command.sh b/t/t0061-run-command.sh index dc9d77b0e14ab4..c74deaee1ac8fb 100755 --- a/t/t0061-run-command.sh +++ b/t/t0061-run-command.sh @@ -164,6 +164,77 @@ test_expect_success 'run_command runs ungrouped in parallel with more tasks than test_line_count = 4 err ' +wait_for_line_count () { + expected=$1 && + file=$2 && + + for i in $(test_seq 1 100) + do + if test "$(wc -l <"$file")" -eq "$expected" + then + return 0 + fi && + sleep 0.1 + done && + return 1 +} + +cleanup_parallel () { + touch release + if test -n "$parallel_pid" + then + wait "$parallel_pid" + fi +} + +test_expect_success MINGW 'setup poll descriptor limit test' ' + write_script wait-for-release <<-\EOF + echo started >>"$1" + if test "$3" = stdin + then + while read line + do + : + done + fi + while ! test -e "$2" + do + sleep 0.1 + done + EOF +' + +test_expect_success MINGW 'run_command uses full poll limit without stdin' ' + : >started && + rm -f release && + test-tool run-command run-command-parallel --tasks=40 40 \ + ./wait-for-release "$PWD/started" "$PWD/release" \ + >out 2>err & + parallel_pid=$! && + test_when_finished cleanup_parallel && + wait_for_line_count 40 started && + touch release && + wait "$parallel_pid" && + parallel_pid= +' + +test_expect_success MINGW 'run_command limits children with stdin pipes' ' + : >started && + rm -f release && + test-tool run-command run-command-stdin --tasks=40 40 \ + ./wait-for-release "$PWD/started" "$PWD/release" stdin \ + >out 2>err & + parallel_pid=$! && + test_when_finished cleanup_parallel && + wait_for_line_count 31 started && + sleep 1 && + test_line_count = 31 started && + touch release && + wait "$parallel_pid" && + parallel_pid= && + test_line_count = 40 started +' + test_expect_success 'run_command listens to stdin' ' cat >expect <<-\EOF && preloaded output of a child From 2b9972c81ee0c0e5612989eb1014e6488b21f11f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 30 Jun 2017 22:32:33 +0200 Subject: [PATCH 202/229] tests (mingw): remove Bash-specific pwd option The -W option is only understood by MSYS2 Bash's pwd command. We already make sure to override `pwd` by `builtin pwd -W` for MINGW, so let's not double the effort here. This will also help when switching the shell to another one (such as BusyBox' ash) whose pwd does *not* understand the -W option. Signed-off-by: Johannes Schindelin --- t/t9902-completion.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/t/t9902-completion.sh b/t/t9902-completion.sh index f6de74054a816e..25b162e6975b27 100755 --- a/t/t9902-completion.sh +++ b/t/t9902-completion.sh @@ -139,12 +139,7 @@ invalid_variable_name='${foo.bar}' actual="$TRASH_DIRECTORY/actual" -if test_have_prereq MINGW -then - ROOT="$(pwd -W)" -else - ROOT="$(pwd)" -fi +ROOT="$(pwd)" test_expect_success 'setup for __git_find_repo_path/__gitdir tests' ' mkdir -p subdir/subsubdir && From de7d9c72bd9b551e73fb3226ba60e9f2d1e9ce0f Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 19 Jul 2017 17:07:56 +0200 Subject: [PATCH 203/229] test-lib: add BUSYBOX prerequisite When running with BusyBox, we will want to avoid calling executables on the PATH that are implemented in BusyBox itself. Signed-off-by: Johannes Schindelin --- t/test-lib.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/t/test-lib.sh b/t/test-lib.sh index 9ddb166d12f161..84f3a4bec6c15b 100644 --- a/t/test-lib.sh +++ b/t/test-lib.sh @@ -1939,6 +1939,10 @@ test_lazy_prereq UNZIP ' test $? -ne 127 ' +test_lazy_prereq BUSYBOX ' + case "$($SHELL --help 2>&1)" in *BusyBox*) true;; *) false;; esac +' + run_with_limited_cmdline () { (ulimit -s 128 && "$@") } From b1d01b021b01c0dbb604153e104ab606e0895287 Mon Sep 17 00:00:00 2001 From: Doug Kelly Date: Wed, 8 Jan 2014 20:28:15 -0600 Subject: [PATCH 204/229] pack-objects (mingw): demonstrate a segmentation fault with large deltas There is a problem in the way 9ac3f0e5b3e4 (pack-objects: fix performance issues on packing large deltas, 2018-07-22) initializes that mutex in the `packing_data` struct. The problem manifests in a segmentation fault on Windows, when a mutex (AKA critical section) is accessed without being initialized. (With pthreads, you apparently do not really have to initialize them?) This was reported in https://github.com/git-for-windows/git/issues/1839. Signed-off-by: Doug Kelly Signed-off-by: Johannes Schindelin --- t/meson.build | 1 + t/t7429-submodule-long-path.sh | 106 +++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100755 t/t7429-submodule-long-path.sh diff --git a/t/meson.build b/t/meson.build index e352ca353a09d5..7cb7815ad26086 100644 --- a/t/meson.build +++ b/t/meson.build @@ -933,6 +933,7 @@ integration_tests = [ 't7424-submodule-mixed-ref-formats.sh', 't7425-submodule-gitdir-path-extension.sh', 't7426-submodule-get-default-remote.sh', + 't7429-submodule-long-path.sh', 't7450-bad-git-dotfiles.sh', 't7500-commit-template-squash-signoff.sh', 't7501-commit-basic-functionality.sh', diff --git a/t/t7429-submodule-long-path.sh b/t/t7429-submodule-long-path.sh new file mode 100755 index 00000000000000..f692cedbff7ff8 --- /dev/null +++ b/t/t7429-submodule-long-path.sh @@ -0,0 +1,106 @@ +#!/bin/sh +# +# Copyright (c) 2013 Doug Kelly +# + +test_description='Test submodules with a path near PATH_MAX + +This test verifies that "git submodule" initialization, update and clones work, including with recursive submodules and paths approaching PATH_MAX (260 characters on Windows) +' + +TEST_NO_CREATE_REPO=1 +. ./test-lib.sh + +longpath="" +for (( i=0; i<4; i++ )); do + longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" +done +# Pick a substring maximum of 90 characters +# This should be good, since we'll add on a lot for temp directories +longpath=${longpath:0:90}; export longpath + +test_expect_failure 'submodule with a long path' ' + git config --global protocol.file.allow always && + GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ + git -c init.defaultBranch=long init --bare remote && + test_create_repo bundle1 && + ( + cd bundle1 && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + mkdir home && + ( + cd home && + git clone ../remote test && + cd test && + git checkout -B long && + git submodule add ../bundle1 $longpath && + test_commit "sogood" && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) && + git push origin long + ) && + mkdir home2 && + ( + cd home2 && + git clone ../remote test && + cd test && + git checkout long && + git submodule update --init && + ( + cd $longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../expect actual + ) + ) +' + +test_expect_failure 'recursive submodule with a long path' ' + GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ + git -c init.defaultBranch=long init --bare super && + test_create_repo child && + ( + cd child && + test_commit "shoot" && + git rev-parse --verify HEAD >../expect + ) && + test_create_repo parent && + ( + cd parent && + git submodule add ../child $longpath && + test_commit "aim" + ) && + mkdir home3 && + ( + cd home3 && + git clone ../super test && + cd test && + git checkout -B long && + git submodule add ../parent foo && + git submodule update --init --recursive && + test_commit "sogood" && + ( + cd foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) && + git push origin long + ) && + mkdir home4 && + ( + cd home4 && + git clone ../super test --recursive && + ( + cd test/foo/$longpath && + git rev-parse --verify HEAD >actual && + test_cmp ../../../../expect actual + ) + ) +' +unset longpath + +test_done From 770be31c615f680f05c8c77983d849f7e6ddc170 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sat, 5 Aug 2017 21:36:01 +0200 Subject: [PATCH 205/229] t5003: use binary file from t/lib-diff/ At some stage, t5003-archive-zip wants to add a file that is not ASCII. To that end, it uses /bin/sh. But that file may actually not exist (it is too easy to forget that not all the world is Unix/Linux...)! Besides, we already have perfectly fine binary files intended for use solely by the tests. So let's use one of them instead. Signed-off-by: Johannes Schindelin --- t/t5003-archive-zip.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t5003-archive-zip.sh b/t/t5003-archive-zip.sh index c8c1c5c06b6037..8f2a2cbc6b8103 100755 --- a/t/t5003-archive-zip.sh +++ b/t/t5003-archive-zip.sh @@ -88,7 +88,7 @@ test_expect_success \ 'mkdir a && echo simple textfile >a/a && mkdir a/bin && - cp /bin/sh a/bin && + cp "$TEST_DIRECTORY/lib-diff/test-binary-1.png" a/bin && printf "text\r" >a/text.cr && printf "text\r\n" >a/text.crlf && printf "text\n" >a/text.lf && From 37bc17533b9c427aafcb80b8b1e50ee852d6eff3 Mon Sep 17 00:00:00 2001 From: Karsten Blees Date: Tue, 28 Jul 2015 21:07:41 +0200 Subject: [PATCH 206/229] mingw: support long paths Windows paths are typically limited to MAX_PATH = 260 characters, even though the underlying NTFS file system supports paths up to 32,767 chars. This limitation is also evident in Windows Explorer, cmd.exe and many other applications (including IDEs). Particularly annoying is that most Windows APIs return bogus error codes if a relative path only barely exceeds MAX_PATH in conjunction with the current directory, e.g. ERROR_PATH_NOT_FOUND / ENOENT instead of the infinitely more helpful ERROR_FILENAME_EXCED_RANGE / ENAMETOOLONG. Many Windows wide char APIs support longer than MAX_PATH paths through the file namespace prefix ('\\?\' or '\\?\UNC\') followed by an absolute path. Notable exceptions include functions dealing with executables and the current directory (CreateProcess, LoadLibrary, Get/SetCurrentDirectory) as well as the entire shell API (ShellExecute, SHGetSpecialFolderPath...). Introduce a handle_long_path function to check the length of a specified path properly (and fail with ENAMETOOLONG), and to optionally expand long paths using the '\\?\' file namespace prefix. Short paths will not be modified, so we don't need to worry about device names (NUL, CON, AUX). Contrary to MSDN docs, the GetFullPathNameW function doesn't seem to be limited to MAX_PATH (at least not on Win7), so we can use it to do the heavy lifting of the conversion (translate '/' to '\', eliminate '.' and '..', and make an absolute path). Add long path error checking to xutftowcs_path for APIs with hard MAX_PATH limit. Add a new MAX_LONG_PATH constant and xutftowcs_long_path function for APIs that support long paths. While improved error checking is always active, long paths support must be explicitly enabled via 'core.longpaths' option. This is to prevent end users from shooting themselves in the foot by checking out files that Windows Explorer, cmd/bash or their favorite IDE cannot handle. Test suite: Test the case is when the full pathname length of a dir is close to 260 (MAX_PATH). Bug report and an original reproducer by Andrey Rogozhnikov: https://github.com/msysgit/git/pull/122#issuecomment-43604199 [jes: adjusted test number to avoid conflicts, added support for chdir(), etc] Thanks-to: Martin W. Kirst Thanks-to: Doug Kelly Original-test-by: Andrey Rogozhnikov Signed-off-by: Karsten Blees Signed-off-by: Stepan Kasal Signed-off-by: Johannes Schindelin Signed-off-by: Josh Soref --- Documentation/config/core.adoc | 7 ++ compat/mingw.c | 204 ++++++++++++++++++++++++--------- compat/mingw.h | 75 +++++++++++- compat/win32/dirent.c | 17 ++- compat/win32/fscache.c | 22 ++-- setup.c | 4 +- t/meson.build | 1 + t/t2031-checkout-long-paths.sh | 102 +++++++++++++++++ t/t7429-submodule-long-path.sh | 24 ++-- 9 files changed, 368 insertions(+), 88 deletions(-) create mode 100755 t/t2031-checkout-long-paths.sh diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index ca8da41aa50fdf..1e952286c85275 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -751,6 +751,13 @@ core.fscache:: Git for Windows uses this to bulk-read and cache lstat data of entire directories (instead of doing lstat file by file). +core.longpaths:: + Enable long path (> 260) support for builtin commands in Git for + Windows. This is disabled by default, as long paths are not supported + by Windows Explorer, cmd.exe and the Git for Windows tool chain + (msys, bash, tcl, perl...). Only enable this if you know what you're + doing and are prepared to live with a few quirks. + core.unsetenvvars:: Windows-only: comma-separated list of environment variables' names that need to be unset before spawning any other process. diff --git a/compat/mingw.c b/compat/mingw.c index 69896770fc7a8f..2a12bb90fc467a 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -281,6 +281,27 @@ static enum hide_dotfiles_type hide_dotfiles = HIDE_DOTFILES_DOTGITONLY; static char *unset_environment_variables; int core_fscache; +int are_long_paths_enabled(void) +{ + /* default to `false` during initialization */ + static const int fallback = 0; + + static int enabled = -1; + + if (enabled < 0) { + /* avoid infinite recursion */ + if (!the_repository) + return fallback; + + if (the_repository->config && + the_repository->config->hash_initialized && + repo_config_get_bool(the_repository, "core.longpaths", &enabled) < 0) + enabled = 0; + } + + return enabled < 0 ? fallback : enabled; +} + int mingw_core_config(const char *var, const char *value, const struct config_context *ctx UNUSED, void *cb UNUSED) @@ -361,7 +382,7 @@ process_phantom_symlink(const wchar_t *wtarget, const wchar_t *wlink) { HANDLE hnd; BY_HANDLE_FILE_INFORMATION fdata; - wchar_t relative[MAX_PATH]; + wchar_t relative[MAX_LONG_PATH]; const wchar_t *rel; /* @@ -576,8 +597,8 @@ int mingw_unlink(const char *pathname, int handle_in_use_error) { static int use_legacy_delete = -1; int tries = 0; - wchar_t wpathname[MAX_PATH]; - if (xutftowcs_path(wpathname, pathname) < 0) + wchar_t wpathname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; if (use_legacy_delete < 0) @@ -612,7 +633,7 @@ static int is_dir_empty(const wchar_t *wpath) { WIN32_FIND_DATAW findbuf; HANDLE handle; - wchar_t wbuf[MAX_PATH + 2]; + wchar_t wbuf[MAX_LONG_PATH + 2]; wcscpy(wbuf, wpath); wcscat(wbuf, L"\\*"); handle = FindFirstFileW(wbuf, &findbuf); @@ -633,7 +654,7 @@ static int is_dir_empty(const wchar_t *wpath) int mingw_rmdir(const char *pathname) { int tries = 0; - wchar_t wpathname[MAX_PATH]; + wchar_t wpathname[MAX_LONG_PATH]; struct stat st; /* @@ -655,7 +676,7 @@ int mingw_rmdir(const char *pathname) return -1; } - if (xutftowcs_path(wpathname, pathname) < 0) + if (xutftowcs_long_path(wpathname, pathname) < 0) return -1; do { @@ -724,15 +745,18 @@ static int set_hidden_flag(const wchar_t *path, int set) int mingw_mkdir(const char *path, int mode UNUSED) { int ret; - wchar_t wpath[MAX_PATH]; + wchar_t wpath[MAX_LONG_PATH]; if (!is_valid_win32_path(path, 0)) { errno = EINVAL; return -1; } - if (xutftowcs_path(wpath, path) < 0) + /* CreateDirectoryW path limit is 248 (MAX_PATH - 8.3 file name) */ + if (xutftowcs_path_ex(wpath, path, MAX_LONG_PATH, -1, 248, + are_long_paths_enabled()) < 0) return -1; + ret = _wmkdir(wpath); if (!ret) process_phantom_symlinks(); @@ -898,7 +922,7 @@ int mingw_open (const char *filename, int oflags, ...) va_list args; unsigned mode; int fd, create = (oflags & (O_CREAT | O_EXCL)) == (O_CREAT | O_EXCL); - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; open_fn_t open_fn; WIN32_FILE_ATTRIBUTE_DATA fdata; @@ -931,7 +955,7 @@ int mingw_open (const char *filename, int oflags, ...) if (filename && !strcmp(filename, "/dev/null")) wcscpy(wfilename, L"nul"); - else if (xutftowcs_path(wfilename, filename) < 0) + else if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* @@ -1022,14 +1046,14 @@ FILE *mingw_fopen (const char *filename, const char *otype) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) wcscpy(wfilename, L"nul"); else if (!is_valid_win32_path(filename, 1)) { int create = otype && strchr(otype, 'w'); errno = create ? EINVAL : ENOENT; return NULL; - } else if (xutftowcs_path(wfilename, filename) < 0) + } else if (xutftowcs_long_path(wfilename, filename) < 0) return NULL; if (xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) @@ -1051,14 +1075,14 @@ FILE *mingw_freopen (const char *filename, const char *otype, FILE *stream) { int hide = needs_hiding(filename); FILE *file; - wchar_t wfilename[MAX_PATH], wotype[4]; + wchar_t wfilename[MAX_LONG_PATH], wotype[4]; if (filename && !strcmp(filename, "/dev/null")) wcscpy(wfilename, L"nul"); else if (!is_valid_win32_path(filename, 1)) { int create = otype && strchr(otype, 'w'); errno = create ? EINVAL : ENOENT; return NULL; - } else if (xutftowcs_path(wfilename, filename) < 0) + } else if (xutftowcs_long_path(wfilename, filename) < 0) return NULL; if (xutftowcs(wotype, otype, ARRAY_SIZE(wotype)) < 0) @@ -1108,7 +1132,7 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) HANDLE h = (HANDLE) _get_osfhandle(fd); if (GetFileType(h) != FILE_TYPE_PIPE) { if (orig == EINVAL) { - wchar_t path[MAX_PATH]; + wchar_t path[MAX_LONG_PATH]; DWORD ret = GetFinalPathNameByHandleW(h, path, ARRAY_SIZE(path), 0); UINT drive_type = ret > 0 && ret < ARRAY_SIZE(path) ? @@ -1145,20 +1169,23 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) int mingw_access(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; if (!strcmp("nul", filename) || !strcmp("/dev/null", filename)) return 0; - if (xutftowcs_path(wfilename, filename) < 0) + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; /* X_OK is not supported by the MSVCRT version */ return _waccess(wfilename, mode & ~X_OK); } +/* cached length of current directory for handle_long_path */ +static int current_directory_len = 0; + int mingw_chdir(const char *dirname) { - wchar_t wdirname[MAX_PATH]; - - if (xutftowcs_path(wdirname, dirname) < 0) + int result; + wchar_t wdirname[MAX_LONG_PATH]; + if (xutftowcs_long_path(wdirname, dirname) < 0) return -1; if (repo_has_symlinks(the_repository)) { @@ -1177,13 +1204,15 @@ int mingw_chdir(const char *dirname) CloseHandle(hnd); } - return _wchdir(normalize_ntpath(wdirname)); + result = _wchdir(normalize_ntpath(wdirname)); + current_directory_len = GetCurrentDirectoryW(0, NULL); + return result; } int mingw_chmod(const char *filename, int mode) { - wchar_t wfilename[MAX_PATH]; - if (xutftowcs_path(wfilename, filename) < 0) + wchar_t wfilename[MAX_LONG_PATH]; + if (xutftowcs_long_path(wfilename, filename) < 0) return -1; return _wchmod(wfilename, mode); } @@ -1317,8 +1346,8 @@ int mingw_lstat(const char *file_name, struct stat *buf) WIN32_FILE_ATTRIBUTE_DATA fdata; DWORD reparse_tag = 0; int link_len = 0; - wchar_t wfilename[MAX_PATH]; - int wlen = xutftowcs_path(wfilename, file_name); + wchar_t wfilename[MAX_LONG_PATH]; + int wlen = xutftowcs_long_path(wfilename, file_name); if (wlen < 0) return -1; @@ -1333,7 +1362,7 @@ int mingw_lstat(const char *file_name, struct stat *buf) if (GetFileAttributesExW(wfilename, GetFileExInfoStandard, &fdata)) { /* for reparse points, get the link tag and length */ if (fdata.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - char tmpbuf[MAX_PATH]; + char tmpbuf[MAX_LONG_PATH]; if (read_reparse_point(wfilename, FALSE, tmpbuf, &link_len, &reparse_tag) < 0) @@ -1414,12 +1443,12 @@ static int get_file_info_by_handle(HANDLE hnd, struct stat *buf) int mingw_stat(const char *file_name, struct stat *buf) { - wchar_t wfile_name[MAX_PATH]; + wchar_t wfile_name[MAX_LONG_PATH]; HANDLE hnd; int result; /* open the file and let Windows resolve the links */ - if (xutftowcs_path(wfile_name, file_name) < 0) + if (xutftowcs_long_path(wfile_name, file_name) < 0) return -1; hnd = CreateFileW(wfile_name, 0, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, @@ -1487,10 +1516,10 @@ int mingw_utime (const char *file_name, const struct utimbuf *times) FILETIME mft, aft; int rc; DWORD attrs; - wchar_t wfilename[MAX_PATH]; + wchar_t wfilename[MAX_LONG_PATH]; HANDLE osfilehandle; - if (xutftowcs_path(wfilename, file_name) < 0) + if (xutftowcs_long_path(wfilename, file_name) < 0) return -1; /* must have write permission */ @@ -1631,14 +1660,14 @@ struct tm *localtime_r(const time_t *timep, struct tm *result) char *mingw_strbuf_realpath(struct strbuf *resolved, const char *path) { - wchar_t wpath[MAX_PATH]; + wchar_t wpath[MAX_LONG_PATH]; HANDLE h; DWORD ret; int len; const char *last_component = NULL; char *append = NULL; - if (xutftowcs_path(wpath, path) < 0) + if (xutftowcs_long_path(wpath, path) < 0) return NULL; h = CreateFileW(wpath, 0, @@ -2198,6 +2227,10 @@ static pid_t mingw_spawnve_fd(const char *cmd, const char **argv, char **deltaen if (*argv && !strcmp(cmd, *argv)) wcmd[0] = L'\0'; + /* + * Paths to executables and to the current directory do not support + * long paths, therefore we cannot use xutftowcs_long_path() here. + */ else if (xutftowcs_path(wcmd, cmd) < 0) return -1; if (dir && xutftowcs_path(wdir, dir) < 0) @@ -2907,12 +2940,12 @@ int mingw_rename(const char *pold, const char *pnew) static int supports_file_rename_info_ex = 1; DWORD attrs = INVALID_FILE_ATTRIBUTES, gle, attrsold; int tries = 0; - wchar_t wpold[MAX_PATH], wpnew[MAX_PATH]; + wchar_t wpold[MAX_LONG_PATH], wpnew[MAX_LONG_PATH]; int wpnew_len; - if (xutftowcs_path(wpold, pold) < 0) + if (xutftowcs_long_path(wpold, pold) < 0) return -1; - wpnew_len = xutftowcs_path(wpnew, pnew); + wpnew_len = xutftowcs_long_path(wpnew, pnew); if (wpnew_len < 0) return -1; @@ -2942,9 +2975,9 @@ int mingw_rename(const char *pold, const char *pnew) * flex array so that the structure has to be allocated on * the heap. As we declare this structure ourselves though * we can avoid the allocation and define FileName to have - * MAX_PATH bytes. + * MAX_LONG_PATH bytes. */ - WCHAR FileName[MAX_PATH]; + WCHAR FileName[MAX_LONG_PATH]; } rename_info = { 0 }; HANDLE old_handle = INVALID_HANDLE_VALUE; BOOL success; @@ -3318,9 +3351,9 @@ int mingw_raise(int sig) int link(const char *oldpath, const char *newpath) { - wchar_t woldpath[MAX_PATH], wnewpath[MAX_PATH]; - if (xutftowcs_path(woldpath, oldpath) < 0 || - xutftowcs_path(wnewpath, newpath) < 0) + wchar_t woldpath[MAX_LONG_PATH], wnewpath[MAX_LONG_PATH]; + if (xutftowcs_long_path(woldpath, oldpath) < 0 || + xutftowcs_long_path(wnewpath, newpath) < 0) return -1; if (!CreateHardLinkW(wnewpath, woldpath, NULL)) { @@ -3363,7 +3396,7 @@ static enum symlink_type check_symlink_attr(struct index_state *index, const cha int mingw_create_symlink(struct index_state *index, const char *target, const char *link) { - wchar_t wtarget[MAX_PATH], wlink[MAX_PATH]; + wchar_t wtarget[MAX_LONG_PATH], wlink[MAX_LONG_PATH]; int len; /* fail if symlinks are disabled or API is not supported (WinXP) */ @@ -3372,8 +3405,8 @@ int mingw_create_symlink(struct index_state *index, const char *target, const ch return -1; } - if ((len = xutftowcs_path(wtarget, target)) < 0 - || xutftowcs_path(wlink, link) < 0) + if ((len = xutftowcs_long_path(wtarget, target)) < 0 + || xutftowcs_long_path(wlink, link) < 0) return -1; /* convert target dir separators to backslashes */ @@ -3410,12 +3443,12 @@ int mingw_create_symlink(struct index_state *index, const char *target, const ch int readlink(const char *path, char *buf, size_t bufsiz) { - WCHAR wpath[MAX_PATH]; - char tmpbuf[MAX_PATH]; + WCHAR wpath[MAX_LONG_PATH]; + char tmpbuf[MAX_LONG_PATH]; int len; DWORD tag; - if (xutftowcs_path(wpath, path) < 0) + if (xutftowcs_long_path(wpath, path) < 0) return -1; if (read_reparse_point(wpath, TRUE, tmpbuf, &len, &tag) < 0) @@ -3490,8 +3523,8 @@ int mingw_is_mount_point(struct strbuf *path) { WIN32_FIND_DATAW findbuf = { 0 }; HANDLE handle; - wchar_t wfilename[MAX_PATH]; - int wlen = xutftowcs_path(wfilename, path->buf); + wchar_t wfilename[MAX_LONG_PATH]; + int wlen = xutftowcs_long_path(wfilename, path->buf); if (wlen < 0) die(_("could not get long path for '%s'"), path->buf); @@ -3634,9 +3667,9 @@ static size_t append_system_bin_dirs(char *path, size_t size) static int is_system32_path(const char *path) { - WCHAR system32[MAX_PATH], wpath[MAX_PATH]; + WCHAR system32[MAX_LONG_PATH], wpath[MAX_LONG_PATH]; - if (xutftowcs_path(wpath, path) < 0 || + if (xutftowcs_long_path(wpath, path) < 0 || !GetSystemDirectoryW(system32, ARRAY_SIZE(system32)) || _wcsicmp(system32, wpath)) return 0; @@ -3860,16 +3893,16 @@ int is_valid_windows_path_element(wchar_t ch) int is_path_owned_by_current_sid(const char *path, struct strbuf *report) { - WCHAR wpath[MAX_PATH]; + WCHAR wpath[MAX_LONG_PATH]; PSID sid = NULL; PSECURITY_DESCRIPTOR descriptor = NULL; DWORD err; - static wchar_t home[MAX_PATH]; + static wchar_t home[MAX_LONG_PATH]; int result = 0; - if (xutftowcs_path(wpath, path) < 0) + if (xutftowcs_long_path(wpath, path) < 0) return 0; /* @@ -4113,6 +4146,68 @@ int is_valid_win32_path(const char *path, int allow_literal_nul) } } +int handle_long_path(wchar_t *path, int len, int max_path, int expand) +{ + int result; + wchar_t buf[MAX_LONG_PATH]; + + /* + * we don't need special handling if path is relative to the current + * directory, and current directory + path don't exceed the desired + * max_path limit. This should cover > 99 % of cases with minimal + * performance impact (git almost always uses relative paths). + */ + if ((len < 2 || (!is_dir_sep(path[0]) && path[1] != ':')) && + (current_directory_len + len < max_path)) + return len; + + /* + * handle everything else: + * - absolute paths: "C:\dir\file" + * - absolute UNC paths: "\\server\share\dir\file" + * - absolute paths on current drive: "\dir\file" + * - relative paths on other drive: "X:file" + * - prefixed paths: "\\?\...", "\\.\..." + */ + + /* convert to absolute path using GetFullPathNameW */ + result = GetFullPathNameW(path, MAX_LONG_PATH, buf, NULL); + if (!result) { + errno = err_win_to_posix(GetLastError()); + return -1; + } + + /* + * return absolute path if it fits within max_path (even if + * "cwd + path" doesn't due to '..' components) + */ + if (result < max_path) { + wcscpy(path, buf); + return result; + } + + /* error out if we shouldn't expand the path or buf is too small */ + if (!expand || result >= MAX_LONG_PATH - 6) { + errno = ENAMETOOLONG; + return -1; + } + + /* prefix full path with "\\?\" or "\\?\UNC\" */ + if (buf[0] == '\\') { + /* ...unless already prefixed */ + if (buf[1] == '\\' && (buf[2] == '?' || buf[2] == '.')) + return len; + + wcscpy(path, L"\\\\?\\UNC\\"); + wcscpy(path + 8, buf + 2); + return result + 6; + } else { + wcscpy(path, L"\\\\?\\"); + wcscpy(path + 4, buf); + return result + 4; + } +} + #if !defined(_MSC_VER) /* * Disable MSVCRT command line wildcard expansion (__getmainargs called from @@ -4304,6 +4399,9 @@ int wmain(int argc, const wchar_t **wargv) /* initialize Unicode console */ winansi_init(); + /* init length of current directory for handle_long_path */ + current_directory_len = GetCurrentDirectoryW(0, NULL); + /* invoke the real main() using our utf8 version of argv. */ exit_status = main(argc, argv); diff --git a/compat/mingw.h b/compat/mingw.h index 4bbc6c5ce78759..379d7f5d079165 100644 --- a/compat/mingw.h +++ b/compat/mingw.h @@ -1,6 +1,7 @@ #include "mingw-posix.h" extern int core_fscache; +int are_long_paths_enabled(void); struct config_context; int mingw_core_config(const char *var, const char *value, @@ -79,6 +80,42 @@ int is_valid_windows_path_element(wchar_t ch); int is_valid_win32_path(const char *path, int allow_literal_nul); #define is_valid_path(path) is_valid_win32_path(path, 0) +/** + * Max length of long paths (exceeding MAX_PATH). The actual maximum supported + * by NTFS is 32,767 (* sizeof(wchar_t)), but we choose an arbitrary smaller + * value to limit required stack memory. + */ +#define MAX_LONG_PATH 4096 + +/** + * Handles paths that would exceed the MAX_PATH limit of Windows Unicode APIs. + * + * With expand == false, the function checks for over-long paths and fails + * with ENAMETOOLONG. The path parameter is not modified, except if cwd + path + * exceeds max_path, but the resulting absolute path doesn't (e.g. due to + * eliminating '..' components). The path parameter must point to a buffer + * of max_path wide characters. + * + * With expand == true, an over-long path is automatically converted in place + * to an absolute path prefixed with '\\?\', and the new length is returned. + * The path parameter must point to a buffer of MAX_LONG_PATH wide characters. + * + * Parameters: + * path: path to check and / or convert + * len: size of path on input (number of wide chars without \0) + * max_path: max short path length to check (usually MAX_PATH = 260, but just + * 248 for CreateDirectoryW) + * expand: false to only check the length, true to expand the path to a + * '\\?\'-prefixed absolute path + * + * Return: + * length of the resulting path, or -1 on failure + * + * Errors: + * ENAMETOOLONG if path is too long + */ +int handle_long_path(wchar_t *path, int len, int max_path, int expand); + /** * Converts UTF-8 encoded string to UTF-16LE. * @@ -137,18 +174,46 @@ static inline int xutftowcs(wchar_t *wcs, const char *utf, size_t wcslen) } /** - * Simplified file system specific variant of xutftowcsn, assumes output - * buffer size is MAX_PATH wide chars and input string is \0-terminated, - * fails with ENAMETOOLONG if input string is too long. + * Simplified file system specific wrapper of xutftowcsn and handle_long_path. + * Converts ERANGE to ENAMETOOLONG. If expand is true, wcs must be at least + * MAX_LONG_PATH wide chars (see handle_long_path). */ -static inline int xutftowcs_path(wchar_t *wcs, const char *utf) +static inline int xutftowcs_path_ex(wchar_t *wcs, const char *utf, + size_t wcslen, int utflen, int max_path, int expand) { - int result = xutftowcsn(wcs, utf, MAX_PATH, -1); + int result = xutftowcsn(wcs, utf, wcslen, utflen); if (result < 0 && errno == ERANGE) errno = ENAMETOOLONG; + if (result >= 0) + result = handle_long_path(wcs, result, max_path, expand); return result; } +/** + * Simplified file system specific variant of xutftowcsn, assumes output + * buffer size is MAX_PATH wide chars and input string is \0-terminated, + * fails with ENAMETOOLONG if input string is too long. Typically used for + * Windows APIs that don't support long paths, e.g. SetCurrentDirectory, + * LoadLibrary, CreateProcess... + */ +static inline int xutftowcs_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_PATH, -1, MAX_PATH, 0); +} + +/** + * Simplified file system specific variant of xutftowcsn for Windows APIs + * that support long paths via '\\?\'-prefix, assumes output buffer size is + * MAX_LONG_PATH wide chars, fails with ENAMETOOLONG if input string is too + * long. The 'core.longpaths' git-config option controls whether the path + * is only checked or expanded to a long path. + */ +static inline int xutftowcs_long_path(wchar_t *wcs, const char *utf) +{ + return xutftowcs_path_ex(wcs, utf, MAX_LONG_PATH, -1, MAX_PATH, + are_long_paths_enabled()); +} + /** * Converts UTF-16LE encoded string to UTF-8. * diff --git a/compat/win32/dirent.c b/compat/win32/dirent.c index f17e1595468f44..87063101f57202 100644 --- a/compat/win32/dirent.c +++ b/compat/win32/dirent.c @@ -68,19 +68,24 @@ static int dirent_closedir(dirent_DIR *dir) DIR *dirent_opendir(const char *name) { - wchar_t pattern[MAX_PATH + 2]; /* + 2 for '/' '*' */ + wchar_t pattern[MAX_LONG_PATH + 2]; /* + 2 for "\*" */ WIN32_FIND_DATAW fdata; HANDLE h; int len; dirent_DIR *dir; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((len = xutftowcs_path(pattern, name)) < 0) + /* convert name to UTF-16 and check length */ + if ((len = xutftowcs_path_ex(pattern, name, MAX_LONG_PATH, -1, + MAX_PATH - 2, + are_long_paths_enabled())) < 0) return NULL; - /* append optional '/' and wildcard '*' */ + /* + * append optional '\' and wildcard '*'. Note: we need to use '\' as + * Windows doesn't translate '/' to '\' for "\\?\"-prefixed paths. + */ if (len && !is_dir_sep(pattern[len - 1])) - pattern[len++] = '/'; + pattern[len++] = '\\'; pattern[len++] = '*'; pattern[len] = 0; @@ -93,7 +98,7 @@ DIR *dirent_opendir(const char *name) } /* initialize DIR structure and copy first dir entry */ - dir = xmalloc(sizeof(dirent_DIR) + MAX_PATH); + dir = xmalloc(sizeof(dirent_DIR) + MAX_LONG_PATH); dir->base_dir.preaddir = (struct dirent *(*)(DIR *dir)) dirent_readdir; dir->base_dir.pclosedir = (int (*)(DIR *dir)) dirent_closedir; dir->dd_handle = h; diff --git a/compat/win32/fscache.c b/compat/win32/fscache.c index 26ae9ab1c1a464..cbd90ececf6b37 100644 --- a/compat/win32/fscache.c +++ b/compat/win32/fscache.c @@ -85,7 +85,7 @@ struct fsentry { struct heap_fsentry { union { struct fsentry ent; - char dummy[sizeof(struct fsentry) + MAX_PATH]; + char dummy[sizeof(struct fsentry) + MAX_LONG_PATH]; } u; }; #pragma GCC diagnostic pop @@ -129,7 +129,7 @@ static void fsentry_init(struct fsentry *fse, struct fsentry *list, const char *name, size_t len) { fse->list = list; - if (len > MAX_PATH) + if (len > MAX_LONG_PATH) BUG("Trying to allocate fsentry for long path '%.*s'", (int)len, name); memcpy(fse->dirent.d_name, name, len); @@ -234,7 +234,7 @@ static struct fsentry *fseentry_create_entry(struct fscache *cache, fdata->EaSize, buf); fse->dirent.d_type = S_ISREG(fse->st_mode) ? DT_REG : S_ISDIR(fse->st_mode) ? DT_DIR : DT_LNK; - fse->u.s.st_size = S_ISLNK(fse->st_mode) ? MAX_PATH : + fse->u.s.st_size = S_ISLNK(fse->st_mode) ? MAX_LONG_PATH : fdata->EndOfFile.LowPart | (((off_t)fdata->EndOfFile.HighPart) << 32); filetime_to_timespec((FILETIME *)&(fdata->LastAccessTime), @@ -270,7 +270,7 @@ static struct fsentry *fseentry_create_entry(struct fscache *cache, static struct fsentry *fsentry_create_list(struct fscache *cache, const struct fsentry *dir, int *dir_not_found) { - wchar_t pattern[MAX_PATH]; + wchar_t pattern[MAX_LONG_PATH]; NTSTATUS status; IO_STATUS_BLOCK iosb; PFILE_FULL_DIR_INFORMATION di; @@ -281,13 +281,11 @@ static struct fsentry *fsentry_create_list(struct fscache *cache, const struct f *dir_not_found = 0; - /* convert name to UTF-16 and check length < MAX_PATH */ - if ((wlen = xutftowcsn(pattern, dir->dirent.d_name, MAX_PATH, - dir->len)) < 0) { - if (errno == ERANGE) - errno = ENAMETOOLONG; + /* convert name to UTF-16 and check length */ + if ((wlen = xutftowcs_path_ex(pattern, dir->dirent.d_name, + MAX_LONG_PATH, dir->len, MAX_PATH - 2, + are_long_paths_enabled())) < 0) return NULL; - } /* handle CWD */ if (!wlen) { @@ -638,8 +636,8 @@ int fscache_lstat(const char *filename, struct stat *st) * Special case symbolic links: FindFirstFile()/FindNextFile() did not * provide us with the length of the target path. */ - if (fse->u.s.st_size == MAX_PATH && S_ISLNK(fse->st_mode)) { - char buf[MAX_PATH]; + if (fse->u.s.st_size == MAX_LONG_PATH && S_ISLNK(fse->st_mode)) { + char buf[MAX_LONG_PATH]; int len = readlink(filename, buf, sizeof(buf) - 1); if (len > 0) diff --git a/setup.c b/setup.c index 35f9aa326d6629..a26d1b411c41bb 100644 --- a/setup.c +++ b/setup.c @@ -940,11 +940,11 @@ static int ensure_valid_ownership(const char *gitfile, static int is_invalid_dotgit_path(const char *gitfile, const char *potential_gitdir) { - WCHAR wpath[MAX_PATH]; + WCHAR wpath[MAX_LONG_PATH]; struct strbuf dir = STRBUF_INIT; int ret; - if (xutftowcs_path(wpath, potential_gitdir) < 0) + if (xutftowcs_long_path(wpath, potential_gitdir) < 0) return 1; /* Do not leak NTLM hashes, UNC paths etc. are generally problematic */ diff --git a/t/meson.build b/t/meson.build index 7cb7815ad26086..ae113665260bd4 100644 --- a/t/meson.build +++ b/t/meson.build @@ -289,6 +289,7 @@ integration_tests = [ 't2026-checkout-pathspec-file.sh', 't2027-checkout-track.sh', 't2030-unresolve-info.sh', + 't2031-checkout-long-paths.sh', 't2040-checkout-symlink-attr.sh', 't2050-git-dir-relative.sh', 't2060-switch.sh', diff --git a/t/t2031-checkout-long-paths.sh b/t/t2031-checkout-long-paths.sh new file mode 100755 index 00000000000000..bc2be4d5677660 --- /dev/null +++ b/t/t2031-checkout-long-paths.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +test_description='checkout long paths on Windows + +Ensures that Git for Windows can deal with long paths (>260) enabled via core.longpaths' + +. ./test-lib.sh + +if test_have_prereq !MINGW +then + skip_all='skipping MINGW specific long paths test' + test_done +fi + +test_expect_success setup ' + p=longpathxx && # -> 10 + p=$p$p$p$p$p && # -> 50 + p=$p$p$p$p$p && # -> 250 + + path=${p}/longtestfile && # -> 263 (MAX_PATH = 260) + + blob=$(echo foobar | git hash-object -w --stdin) && + + printf "100644 %s 0\t%s\n" "$blob" "$path" | + git update-index --add --index-info && + git commit -m initial -q +' + +test_expect_success 'checkout of long paths without core.longpaths fails' ' + git config core.longpaths false && + test_must_fail git checkout -f 2>error && + test_grep -q "Filename too long" error && + test ! -d longpa* +' + +test_expect_success 'checkout of long paths with core.longpaths works' ' + git config core.longpaths true && + git checkout -f && + test_path_is_file longpa*/longtestfile +' + +test_expect_success 'update of long paths' ' + echo frotz >>$(ls longpa*/longtestfile) && + echo $path > expect && + git ls-files -m > actual && + test_cmp expect actual && + git add $path && + git commit -m second && + git grep "frotz" HEAD -- $path +' + +test_expect_success cleanup ' + # bash cannot delete the trash dir if it contains a long path + # lets help cleaning up (unless in debug mode) + if test -z "$debug" + then + rm -rf longpa~1 + fi +' + +# check that the template used in the test won't be too long: +abspath="$(pwd)"/testdir +test ${#abspath} -gt 230 || +test_set_prereq SHORTABSPATH + +test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' + p=/123456789abcdef/123456789abcdef/123456789abcdef/123456789abc/ef && + p=y$p$p$p$p && + subdir="x$(echo "$p" | tail -c $((253 - ${#abspath})) - )" && + # Now, $abspath/$subdir has exactly 254 characters, and is inside CWD + p2="$abspath/$subdir" && + test 254 = ${#p2} && + + # Be careful to overcome path limitations of the MSys tools and split + # the $subdir into two parts. ($subdir2 has to contain 16 chars and a + # slash somewhere following; that is why we asked for abspath <= 230 and + # why we placed a slash near the end of the $subdir template.) + subdir2=${subdir#????????????????*/} && + subdir1=testdir/${subdir%/$subdir2} && + mkdir -p "$subdir1" && + i=0 && + # The most important case is when absolute path is 258 characters long, + # and that will be when i == 4. + while test $i -le 7 + do + mkdir -p $subdir2 && + touch $subdir2/one-file && + mv ${subdir2%%/*} "$subdir1/" && + subdir2=z${subdir2} && + i=$(($i+1)) || + exit 1 + done && + + # now check that git is able to clear the tree: + (cd testdir && + git init && + git config core.longpaths yes && + git clean -fdx) && + test ! -d "$subdir1" +' + +test_done diff --git a/t/t7429-submodule-long-path.sh b/t/t7429-submodule-long-path.sh index f692cedbff7ff8..458519eafd6f03 100755 --- a/t/t7429-submodule-long-path.sh +++ b/t/t7429-submodule-long-path.sh @@ -11,15 +11,20 @@ This test verifies that "git submodule" initialization, update and clones work, TEST_NO_CREATE_REPO=1 . ./test-lib.sh -longpath="" -for (( i=0; i<4; i++ )); do - longpath="0123456789abcdefghijklmnopqrstuvwxyz$longpath" -done -# Pick a substring maximum of 90 characters -# This should be good, since we'll add on a lot for temp directories -longpath=${longpath:0:90}; export longpath +# cloning a submodule calls is_git_directory("$path/../.git/modules/$path"), +# which effectively limits the maximum length to PATH_MAX / 2 minus some +# overhead; start with 3 * 36 = 108 chars (test 2 fails if >= 110) +longpath36=0123456789abcdefghijklmnopqrstuvwxyz +longpath180=$longpath36$longpath36$longpath36$longpath36$longpath36 -test_expect_failure 'submodule with a long path' ' +# the git database must fit within PATH_MAX, which limits the submodule name +# to PATH_MAX - len(pwd) - ~90 (= len("/objects//") + 40-byte sha1 + some +# overhead from the test case) +pwd=$(pwd) +pwdlen=$(echo "$pwd" | wc -c) +longpath=$(echo $longpath180 | cut -c 1-$((170-$pwdlen))) + +test_expect_success 'submodule with a long path' ' git config --global protocol.file.allow always && GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ git -c init.defaultBranch=long init --bare remote && @@ -59,7 +64,7 @@ test_expect_failure 'submodule with a long path' ' ) ' -test_expect_failure 'recursive submodule with a long path' ' +test_expect_success 'recursive submodule with a long path' ' GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME= \ git -c init.defaultBranch=long init --bare super && test_create_repo child && @@ -101,6 +106,5 @@ test_expect_failure 'recursive submodule with a long path' ' ) ) ' -unset longpath test_done From b19bc6028f8d52d051d040d9debdb2849c714c1a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 21 Jul 2017 12:48:33 +0200 Subject: [PATCH 207/229] t5532: workaround for BusyBox on Windows While it may seem super convenient to some old Unix hands to simpy require Perl to be available when running the test suite, this is a major hassle on Windows, where we want to verify that Perl is not, actually, required in a NO_PERL build. As a super ugly workaround, we "install" a script into /usr/bin/perl reading like this: #!/bin/sh # We'd much rather avoid requiring Perl altogether when testing # an installed Git. Oh well, that's why we cannot have nice # things. exec c:/git-sdk-64/usr/bin/perl.exe "$@" The problem with that is that BusyBox assumes that the #! line in a script refers to an executable, not to a script. So when it encounters the line #!/usr/bin/perl in t5532's proxy-get-cmd, it barfs. Let's help this situation by simply executing the Perl script with the "interpreter" specified explicitly. Signed-off-by: Johannes Schindelin --- t/t5532-fetch-proxy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/t5532-fetch-proxy.sh b/t/t5532-fetch-proxy.sh index ccbca7c2f28e97..23cec503cb9787 100755 --- a/t/t5532-fetch-proxy.sh +++ b/t/t5532-fetch-proxy.sh @@ -32,7 +32,7 @@ test_expect_success 'setup proxy script' ' write_script proxy <<-\EOF echo >&2 "proxying for $*" - cmd=$(./proxy-get-cmd) + cmd=$("$PERL_PATH" ./proxy-get-cmd) echo >&2 "Running $cmd" exec $cmd EOF From 00821658c69a58ae1eab9f855caa40ec014fb2ff Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 6 Sep 2023 09:14:47 +0200 Subject: [PATCH 208/229] win32(long path support): leave drive-less absolute paths intact When trying to ensure that long paths are handled correctly, we first normalize absolute paths as we encounter them. However, if the path is a so-called "drive-less" absolute path, i.e. if it is relative to the current drive but _does_ start with a directory separator, we would want the normalized path to be such a drive-less absolute path, too. Let's do that, being careful to still include the drive prefix when we need to go through the `\\?\` dance (because there, the drive prefix is absolutely required). This fixes https://github.com/git-for-windows/git/issues/4586. Signed-off-by: Johannes Schindelin --- compat/mingw.c | 7 ++++++- t/t2031-checkout-long-paths.sh | 9 +++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index 2a12bb90fc467a..672171ee8698f0 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -4182,7 +4182,12 @@ int handle_long_path(wchar_t *path, int len, int max_path, int expand) * "cwd + path" doesn't due to '..' components) */ if (result < max_path) { - wcscpy(path, buf); + /* Be careful not to add a drive prefix if there was none */ + if (is_wdir_sep(path[0]) && + !is_wdir_sep(buf[0]) && buf[1] == L':' && is_wdir_sep(buf[2])) + wcscpy(path, buf + 2); + else + wcscpy(path, buf); return result; } diff --git a/t/t2031-checkout-long-paths.sh b/t/t2031-checkout-long-paths.sh index bc2be4d5677660..4278a602cbd09e 100755 --- a/t/t2031-checkout-long-paths.sh +++ b/t/t2031-checkout-long-paths.sh @@ -99,4 +99,13 @@ test_expect_success SHORTABSPATH 'clean up path close to MAX_PATH' ' test ! -d "$subdir1" ' +test_expect_success SYMLINKS_WINDOWS 'leave drive-less, short paths intact' ' + printf "/Program Files" >symlink-target && + symlink_target_oid="$(git hash-object -w --stdin actual && + test_grep " *PF *\\[\\\\Program Files\\]" actual +' + test_done From 5ae0eb639eb1e295b432fa11d2192bb0b4fe7394 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 21 Jul 2017 13:24:55 +0200 Subject: [PATCH 209/229] t5605: special-case hardlink test for BusyBox-w32 When t5605 tries to verify that files are hardlinked (or that they are not), it uses the `-links` option of the `find` utility. BusyBox' implementation does not support that option, and BusyBox-w32's lstat() does not even report the number of hard links correctly (for performance reasons). So let's just switch to a different method that actually works on Windows. Signed-off-by: Johannes Schindelin --- t/t5605-clone-local.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/t/t5605-clone-local.sh b/t/t5605-clone-local.sh index 156362f145b071..a78def3efbca1b 100755 --- a/t/t5605-clone-local.sh +++ b/t/t5605-clone-local.sh @@ -11,6 +11,21 @@ repo_is_hardlinked() { test_line_count = 0 output } +if test_have_prereq MINGW,BUSYBOX +then + # BusyBox' `find` does not support `-links`. Besides, BusyBox-w32's + # lstat() does not report hard links, just like Git's mingw_lstat() + # (from where BusyBox-w32 got its initial implementation). + repo_is_hardlinked() { + for f in $(find "$1/objects" -type f) + do + "$SYSTEMROOT"/system32/fsutil.exe \ + hardlink list $f >links && + test_line_count -gt 1 links || return 1 + done + } +fi + test_expect_success 'preparing origin repository' ' : >file && git add . && git commit -m1 && git clone --bare . a.git && From 6a32c53b5e3e0f46c624624833c67a2ab96f9d4e Mon Sep 17 00:00:00 2001 From: Jeff Hostetler Date: Fri, 25 Mar 2022 16:56:04 -0400 Subject: [PATCH 210/229] compat/fsmonitor/fsm-*-win32: support long paths Update wchar_t buffers to use MAX_LONG_PATH instead of MAX_PATH and call xutftowcs_long_path() in the Win32 backend source files. Signed-off-by: Jeff Hostetler --- compat/fsmonitor/fsm-health-win32.c | 6 +++--- compat/fsmonitor/fsm-listen-win32.c | 18 +++++++++--------- compat/fsmonitor/fsm-path-utils-win32.c | 8 ++++---- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/compat/fsmonitor/fsm-health-win32.c b/compat/fsmonitor/fsm-health-win32.c index 2aa8c219acee4d..4b53360d194105 100644 --- a/compat/fsmonitor/fsm-health-win32.c +++ b/compat/fsmonitor/fsm-health-win32.c @@ -34,7 +34,7 @@ struct fsm_health_data struct wt_moved { - wchar_t wpath[MAX_PATH + 1]; + wchar_t wpath[MAX_LONG_PATH + 1]; BY_HANDLE_FILE_INFORMATION bhfi; } wt_moved; }; @@ -143,8 +143,8 @@ static int has_worktree_moved(struct fsmonitor_daemon_state *state, return 0; case CTX_INIT: - if (xutftowcs_path(data->wt_moved.wpath, - state->path_worktree_watch.buf) < 0) { + if (xutftowcs_long_path(data->wt_moved.wpath, + state->path_worktree_watch.buf) < 0) { error(_("could not convert to wide characters: '%s'"), state->path_worktree_watch.buf); return -1; diff --git a/compat/fsmonitor/fsm-listen-win32.c b/compat/fsmonitor/fsm-listen-win32.c index 9a6efc9bea340b..afcc172750af10 100644 --- a/compat/fsmonitor/fsm-listen-win32.c +++ b/compat/fsmonitor/fsm-listen-win32.c @@ -28,7 +28,7 @@ struct one_watch DWORD count; struct strbuf path; - wchar_t wpath_longname[MAX_PATH + 1]; + wchar_t wpath_longname[MAX_LONG_PATH + 1]; DWORD wpath_longname_len; HANDLE hDir; @@ -131,8 +131,8 @@ static int normalize_path_in_utf8(wchar_t *wpath, DWORD wpath_len, */ static void check_for_shortnames(struct one_watch *watch) { - wchar_t buf_in[MAX_PATH + 1]; - wchar_t buf_out[MAX_PATH + 1]; + wchar_t buf_in[MAX_LONG_PATH + 1]; + wchar_t buf_out[MAX_LONG_PATH + 1]; wchar_t *last; wchar_t *p; @@ -197,8 +197,8 @@ static enum get_relative_result get_relative_longname( const wchar_t *wpath, DWORD wpath_len, wchar_t *wpath_longname, size_t bufsize_wpath_longname) { - wchar_t buf_in[2 * MAX_PATH + 1]; - wchar_t buf_out[MAX_PATH + 1]; + wchar_t buf_in[2 * MAX_LONG_PATH + 1]; + wchar_t buf_out[MAX_LONG_PATH + 1]; DWORD root_len; DWORD out_len; @@ -298,10 +298,10 @@ static struct one_watch *create_watch(const char *path) FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE; HANDLE hDir; DWORD len_longname; - wchar_t wpath[MAX_PATH + 1]; - wchar_t wpath_longname[MAX_PATH + 1]; + wchar_t wpath[MAX_LONG_PATH + 1]; + wchar_t wpath_longname[MAX_LONG_PATH + 1]; - if (xutftowcs_path(wpath, path) < 0) { + if (xutftowcs_long_path(wpath, path) < 0) { error(_("could not convert to wide characters: '%s'"), path); return NULL; } @@ -545,7 +545,7 @@ static int process_worktree_events(struct fsmonitor_daemon_state *state) struct string_list cookie_list = STRING_LIST_INIT_DUP; struct fsmonitor_batch *batch = NULL; const char *p = watch->buffer; - wchar_t wpath_longname[MAX_PATH + 1]; + wchar_t wpath_longname[MAX_LONG_PATH + 1]; /* * If the kernel gets more events than will fit in the kernel diff --git a/compat/fsmonitor/fsm-path-utils-win32.c b/compat/fsmonitor/fsm-path-utils-win32.c index f4f9cc1f336720..c6eb065bde48b4 100644 --- a/compat/fsmonitor/fsm-path-utils-win32.c +++ b/compat/fsmonitor/fsm-path-utils-win32.c @@ -69,8 +69,8 @@ static int check_remote_protocol(wchar_t *wpath) */ int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info) { - wchar_t wpath[MAX_PATH]; - wchar_t wfullpath[MAX_PATH]; + wchar_t wpath[MAX_LONG_PATH]; + wchar_t wfullpath[MAX_LONG_PATH]; size_t wlen; UINT driveType; @@ -78,7 +78,7 @@ int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info) * Do everything in wide chars because the drive letter might be * a multi-byte sequence. See win32_has_dos_drive_prefix(). */ - if (xutftowcs_path(wpath, path) < 0) { + if (xutftowcs_long_path(wpath, path) < 0) { return -1; } @@ -97,7 +97,7 @@ int fsmonitor__get_fs_info(const char *path, struct fs_info *fs_info) * slashes to backslashes. This is essential to get GetDriveTypeW() * correctly handle some UNC "\\server\share\..." paths. */ - if (!GetFullPathNameW(wpath, MAX_PATH, wfullpath, NULL)) { + if (!GetFullPathNameW(wpath, MAX_LONG_PATH, wfullpath, NULL)) { return -1; } From 9d66d623503a91016d57d6e71ce2eae0f5b7cbe1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 5 Jul 2017 15:14:50 +0200 Subject: [PATCH 211/229] t5813: allow for $PWD to be a Windows path Git for Windows uses MSYS2's Bash to run the test suite, which comes with benefits but also at a heavy price: on the plus side, MSYS2's POSIX emulation layer allows us to continue pretending that we are on a Unix system, e.g. use Unix paths instead of Windows ones, yet this is bought at a rather noticeable performance penalty. There *are* some more native ports of Unix shells out there, though, most notably BusyBox-w32's ash. These native ports do not use any POSIX emulation layer (or at most a *very* thin one, choosing to avoid features such as fork() that are expensive to emulate on Windows), and they use native Windows paths (usually with forward slashes instead of backslashes, which is perfectly legal in almost all use cases). And here comes the problem: with a $PWD looking like, say, C:/git-sdk-64/usr/src/git/t/trash directory.t5813-proto-disable-ssh Git's test scripts get quite a bit confused, as their assumptions have been shattered. Not only does this path contain a colon (oh no!), it also does not start with a slash. This is a problem e.g. when constructing a URL as t5813 does it: ssh://remote$PWD. Not only is it impossible to separate the "host" from the path with a $PWD as above, even prefixing $PWD by a slash won't work, as /C:/git-sdk-64/... is not a valid path. As a workaround, detect when $PWD does not start with a slash on Windows, and simply strip the drive prefix, using an obscure feature of Windows paths: if an absolute Windows path starts with a slash, it is implicitly prefixed by the drive prefix of the current directory. As we are talking about the current directory here, anyway, that strategy works. Signed-off-by: Johannes Schindelin --- t/t5813-proto-disable-ssh.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/t/t5813-proto-disable-ssh.sh b/t/t5813-proto-disable-ssh.sh index 1800d27f894414..f0f91718a599df 100755 --- a/t/t5813-proto-disable-ssh.sh +++ b/t/t5813-proto-disable-ssh.sh @@ -15,8 +15,23 @@ test_expect_success 'setup repository to clone' ' ' test_proto "host:path" ssh "remote:repo.git" -test_proto "ssh://" ssh "ssh://remote$PWD/remote/repo.git" -test_proto "git+ssh://" ssh "git+ssh://remote$PWD/remote/repo.git" + +hostdir="$PWD" +if test_have_prereq MINGW && test "/${PWD#/}" != "$PWD" +then + case "$PWD" in + [A-Za-z]:/*) + hostdir="${PWD#?:}" + ;; + *) + skip_all="Unhandled PWD '$PWD'; skipping rest" + test_done + ;; + esac +fi + +test_proto "ssh://" ssh "ssh://remote$hostdir/remote/repo.git" +test_proto "git+ssh://" ssh "git+ssh://remote$hostdir/remote/repo.git" # Don't even bother setting up a "-remote" directory, as ssh would generally # complain about the bogus option rather than completing our request. Our From 93f7be9f8648685dde35dd15daef4eb0166c625c Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Fri, 22 Apr 2022 09:06:23 -0400 Subject: [PATCH 212/229] clean: suggest using `core.longPaths` if paths are too long to remove On Windows, git repositories may have extra files which need cleaned (e.g., a build directory) that may be arbitrarily deep. Suggest using `core.longPaths` if such situations are encountered. Fixes: #2715 Signed-off-by: Ben Boeckel --- Documentation/config/advice.adoc | 3 +++ advice.c | 1 + advice.h | 1 + builtin/clean.c | 13 +++++++++++++ 4 files changed, 18 insertions(+) diff --git a/Documentation/config/advice.adoc b/Documentation/config/advice.adoc index 83039ba48bc797..05edc82626687a 100644 --- a/Documentation/config/advice.adoc +++ b/Documentation/config/advice.adoc @@ -72,6 +72,9 @@ all advice messages. set their identity configuration. mergeConflict:: Shown when various commands stop because of conflicts. + nameTooLong:: + Advice shown if a filepath operation is attempted where the + path was too long. nestedTag:: Shown when a user attempts to recursively tag a tag object. pushAlreadyExists:: diff --git a/advice.c b/advice.c index 7253891f92d791..55b2197a30451f 100644 --- a/advice.c +++ b/advice.c @@ -62,6 +62,7 @@ static struct { [ADVICE_IGNORED_HOOK] = { "ignoredHook" }, [ADVICE_IMPLICIT_IDENTITY] = { "implicitIdentity" }, [ADVICE_MERGE_CONFLICT] = { "mergeConflict" }, + [ADVICE_NAME_TOO_LONG] = { "nameTooLong" }, [ADVICE_NESTED_TAG] = { "nestedTag" }, [ADVICE_OBJECT_NAME_WARNING] = { "objectNameWarning" }, [ADVICE_PUSH_ALREADY_EXISTS] = { "pushAlreadyExists" }, diff --git a/advice.h b/advice.h index a443bc7aebe4e6..76883c968df862 100644 --- a/advice.h +++ b/advice.h @@ -29,6 +29,7 @@ enum advice_type { ADVICE_IGNORED_HOOK, ADVICE_IMPLICIT_IDENTITY, ADVICE_MERGE_CONFLICT, + ADVICE_NAME_TOO_LONG, ADVICE_NESTED_TAG, ADVICE_OBJECT_NAME_WARNING, ADVICE_PUSH_ALREADY_EXISTS, diff --git a/builtin/clean.c b/builtin/clean.c index e15d595c3dc7cc..f8a54a4a47bc7b 100644 --- a/builtin/clean.c +++ b/builtin/clean.c @@ -26,6 +26,7 @@ #include "pathspec.h" #include "help.h" #include "prompt.h" +#include "advice.h" static int require_force = -1; /* unset */ static int interactive; @@ -221,6 +222,9 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, quote_path(path->buf, prefix, "ed, 0); errno = saved_errno; warning_errno(_(msg_warn_remove_failed), quoted.buf); + if (saved_errno == ENAMETOOLONG) { + advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed.")); + } *dir_gone = 0; } ret = res; @@ -256,6 +260,9 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, quote_path(path->buf, prefix, "ed, 0); errno = saved_errno; warning_errno(_(msg_warn_remove_failed), quoted.buf); + if (saved_errno == ENAMETOOLONG) { + advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed.")); + } *dir_gone = 0; ret = 1; } @@ -299,6 +306,9 @@ static int remove_dirs(struct strbuf *path, const char *prefix, int force_flag, quote_path(path->buf, prefix, "ed, 0); errno = saved_errno; warning_errno(_(msg_warn_remove_failed), quoted.buf); + if (saved_errno == ENAMETOOLONG) { + advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed.")); + } *dir_gone = 0; ret = 1; } @@ -1109,6 +1119,9 @@ int cmd_clean(int argc, qname = quote_path(item->string, NULL, &buf, 0); errno = saved_errno; warning_errno(_(msg_warn_remove_failed), qname); + if (saved_errno == ENAMETOOLONG) { + advise_if_enabled(ADVICE_NAME_TOO_LONG, _("Setting `core.longPaths` may allow the deletion to succeed.")); + } errors++; } else if (!quiet) { qname = quote_path(item->string, NULL, &buf, 0); From 1008b8925dff8c0c66d2aecb7e0efdc6eae4e976 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 7 Jul 2017 10:15:36 +0200 Subject: [PATCH 213/229] t9200: skip tests when $PWD contains a colon On Windows, the current working directory is pretty much guaranteed to contain a colon. If we feed that path to CVS, it mistakes it for a separator between host and port, though. This has not been a problem so far because Git for Windows uses MSYS2's Bash using a POSIX emulation layer that also pretends that the current directory is a Unix path (at least as long as we're in a shell script). However, that is rather limiting, as Git for Windows also explores other ports of other Unix shells. One of those is BusyBox-w32's ash, which is a native port (i.e. *not* using any POSIX emulation layer, and certainly not emulating Unix paths). So let's just detect if there is a colon in $PWD and punt in that case. Signed-off-by: Johannes Schindelin --- t/t9200-git-cvsexportcommit.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/t/t9200-git-cvsexportcommit.sh b/t/t9200-git-cvsexportcommit.sh index c42e69ee70ee23..1dada9a414a4aa 100755 --- a/t/t9200-git-cvsexportcommit.sh +++ b/t/t9200-git-cvsexportcommit.sh @@ -11,6 +11,13 @@ if ! test_have_prereq PERL; then test_done fi +case "$PWD" in +*:*) + skip_all='cvs would get confused by the colon in `pwd`; skipping tests' + test_done + ;; +esac + if ! cvs version >/dev/null 2>&1 then skip_all='skipping git cvsexportcommit tests, cvs not found' From 38c9b7d2e6f789f7bc37519316ee506700e3d8eb Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Thu, 25 Nov 2021 11:26:41 +0100 Subject: [PATCH 214/229] Partially un-revert "editor: save and reset terminal after calling EDITOR" In e3f7e01b50be (Revert "editor: save and reset terminal after calling EDITOR", 2021-11-22), we reverted the commit wholesale where the terminal state would be saved and restored before/after calling an editor. The reverted commit was intended to fix a problem with Windows Terminal where simply calling `vi` would cause problems afterwards. To fix the problem addressed by the revert, but _still_ keep the problem with Windows Terminal fixed, let's revert the revert, with a twist: we restrict the save/restore _specifically_ to the case where `vi` (or `vim`) is called, and do not do the same for any other editor. This should still catch the majority of the cases, and will bridge the time until the original patch is re-done in a way that addresses all concerns. Signed-off-by: Johannes Schindelin --- editor.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/editor.c b/editor.c index 0d1cb8768db1a0..36a3f1ec49efea 100644 --- a/editor.c +++ b/editor.c @@ -13,6 +13,7 @@ #include "strvec.h" #include "run-command.h" #include "sigchain.h" +#include "compat/terminal.h" #ifndef DEFAULT_EDITOR #define DEFAULT_EDITOR "vi" @@ -64,6 +65,7 @@ static int launch_specified_editor(const char *editor, const char *path, return error("Terminal is dumb, but EDITOR unset"); if (strcmp(editor, ":")) { + int save_and_restore_term = !strcmp(editor, "vi") || !strcmp(editor, "vim"); struct strbuf realpath = STRBUF_INIT; struct child_process p = CHILD_PROCESS_INIT; int ret, sig; @@ -92,7 +94,11 @@ static int launch_specified_editor(const char *editor, const char *path, strvec_pushv(&p.env, (const char **)env); p.use_shell = 1; p.trace2_child_class = "editor"; + if (save_and_restore_term) + save_and_restore_term = !save_term(1); if (start_command(&p) < 0) { + if (save_and_restore_term) + restore_term(); strbuf_release(&realpath); return error("unable to start editor '%s'", editor); } @@ -100,6 +106,8 @@ static int launch_specified_editor(const char *editor, const char *path, sigchain_push(SIGINT, SIG_IGN); sigchain_push(SIGQUIT, SIG_IGN); ret = finish_command(&p); + if (save_and_restore_term) + restore_term(); strbuf_release(&realpath); sig = ret - 128; sigchain_pop(SIGINT); From 9fe2a97a5bbd0bf5514c14b3882537d1ac8199a0 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 10 Dec 2019 21:41:57 +0100 Subject: [PATCH 215/229] reset: reinstate support for the deprecated --stdin option The `--stdin` option was a well-established paradigm in other commands, therefore we implemented it in `git reset` for use by Visual Studio. Unfortunately, upstream Git decided that it is time to introduce `--pathspec-from-file` instead. To keep backwards-compatibility for some grace period, we therefore reinstate the `--stdin` option on top of the `--pathspec-from-file` option, but mark it firmly as deprecated. Helped-by: Victoria Dye Helped-by: Matthew John Cheetham Signed-off-by: Johannes Schindelin --- Documentation/git-reset.adoc | 11 +++++++++++ builtin/reset.c | 16 ++++++++++++++++ t/meson.build | 1 + t/t7108-reset-stdin.sh | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+) create mode 100755 t/t7108-reset-stdin.sh diff --git a/Documentation/git-reset.adoc b/Documentation/git-reset.adoc index 5023b5069972ca..933e2fac7dd662 100644 --- a/Documentation/git-reset.adoc +++ b/Documentation/git-reset.adoc @@ -12,6 +12,7 @@ git reset [--soft | --mixed [-N] | --hard | --merge | --keep] [-q] [] git reset [-q] [] [--] ... git reset [-q] [--pathspec-from-file= [--pathspec-file-nul]] [] git reset (--patch | -p) [] [--] [...] +DEPRECATED: git reset [-q] [--stdin [-z]] [] DESCRIPTION ----------- @@ -139,6 +140,16 @@ include::diff-context-options.adoc[] + For more details, see the 'pathspec' entry in linkgit:gitglossary[7]. +`--stdin`:: + DEPRECATED (use `--pathspec-from-file=-` instead): Instead of taking + list of paths from the command line, read list of paths from the + standard input. Paths are separated by LF (i.e. one path per line) by + default. + +`-z`:: + DEPRECATED (use `--pathspec-file-nul` instead): Only meaningful with + `--stdin`; paths are separated with NUL character instead of LF. + EXAMPLES -------- diff --git a/builtin/reset.c b/builtin/reset.c index 78e69bd84ba2c3..66c26eab62fc12 100644 --- a/builtin/reset.c +++ b/builtin/reset.c @@ -38,6 +38,8 @@ #include "trace2.h" #include "dir.h" #include "add-interactive.h" +#include "strbuf.h" +#include "quote.h" #define REFRESH_INDEX_DELAY_WARNING_IN_MS (2 * 1000) @@ -46,6 +48,7 @@ static const char * const git_reset_usage[] = { N_("git reset [-q] [] [--] ..."), N_("git reset [-q] [--pathspec-from-file [--pathspec-file-nul]] []"), N_("git reset --patch [] [--] [...]"), + N_("DEPRECATED: git reset [-q] [--stdin [-z]] []"), NULL }; @@ -347,6 +350,7 @@ int cmd_reset(int argc, struct pathspec pathspec; int intent_to_add = 0; struct interactive_options interactive_opts = INTERACTIVE_OPTIONS_INIT; + int nul_term_line = 0, read_from_stdin = 0; const struct option options[] = { OPT__QUIET(&quiet, N_("be quiet, only report errors")), OPT_BOOL(0, "no-refresh", &no_refresh, @@ -379,6 +383,10 @@ int cmd_reset(int argc, N_("record only the fact that removed paths will be added later")), OPT_PATHSPEC_FROM_FILE(&pathspec_from_file), OPT_PATHSPEC_FILE_NUL(&pathspec_file_nul), + OPT_BOOL('z', NULL, &nul_term_line, + N_("DEPRECATED (use --pathspec-file-nul instead): paths are separated with NUL character")), + OPT_BOOL(0, "stdin", &read_from_stdin, + N_("DEPRECATED (use --pathspec-from-file=- instead): read paths from ")), OPT_END() }; @@ -388,6 +396,14 @@ int cmd_reset(int argc, PARSE_OPT_KEEP_DASHDASH); parse_args(&pathspec, argv, prefix, patch_mode, &rev); + if (read_from_stdin) { + warning(_("--stdin is deprecated, please use --pathspec-from-file=- instead")); + free(pathspec_from_file); + pathspec_from_file = xstrdup("-"); + if (nul_term_line) + pathspec_file_nul = 1; + } + if (pathspec_from_file) { if (patch_mode) die(_("options '%s' and '%s' cannot be used together"), "--pathspec-from-file", "--patch"); diff --git a/t/meson.build b/t/meson.build index ff39f526c22f61..ddde845aef49a4 100644 --- a/t/meson.build +++ b/t/meson.build @@ -902,6 +902,7 @@ integration_tests = [ 't7105-reset-patch.sh', 't7106-reset-unborn-branch.sh', 't7107-reset-pathspec-file.sh', + 't7108-reset-stdin.sh', 't7110-reset-merge.sh', 't7111-reset-table.sh', 't7112-reset-submodule.sh', diff --git a/t/t7108-reset-stdin.sh b/t/t7108-reset-stdin.sh new file mode 100755 index 00000000000000..b7cbcbf869296c --- /dev/null +++ b/t/t7108-reset-stdin.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +test_description='reset --stdin' + +. ./test-lib.sh + +test_expect_success 'reset --stdin' ' + test_commit hello && + git rm hello.t && + test -z "$(git ls-files hello.t)" && + echo hello.t | git reset --stdin && + test hello.t = "$(git ls-files hello.t)" +' + +test_expect_success 'reset --stdin -z' ' + test_commit world && + git rm hello.t world.t && + test -z "$(git ls-files hello.t world.t)" && + printf world.tQworld.tQhello.tQ | q_to_nul | git reset --stdin -z && + printf "hello.t\nworld.t\n" >expect && + git ls-files >actual && + test_cmp expect actual +' + +test_expect_success '--stdin requires --mixed' ' + echo hello.t >list && + test_must_fail git reset --soft --stdin Date: Tue, 29 Sep 2020 13:50:59 +0200 Subject: [PATCH 216/229] Add a GitHub workflow to monitor component updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rather than using private IFTTT Applets that send mails to this maintainer whenever a new version of a Git for Windows component was released, let's use the power of GitHub workflows to make this process publicly visible. This workflow monitors the Atom/RSS feeds, and opens a ticket whenever a new version was released. Note: Bash sometimes releases multiple patched versions within a few minutes of each other (i.e. 5.1p1 through 5.1p4, 5.0p15 and 5.0p16). The MSYS2 runtime also has a similar system. We can address those patches as a group, so we shouldn't get multiple issues about them. Note further: We're not acting on newlib releases, OpenSSL alphas, Perl release candidates or non-stable Perl releases. There's no need to open issues about them. Co-authored-by: Matthias Aßhauer Signed-off-by: Johannes Schindelin --- .github/workflows/monitor-components.yml | 91 ++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/monitor-components.yml diff --git a/.github/workflows/monitor-components.yml b/.github/workflows/monitor-components.yml new file mode 100644 index 00000000000000..e9f16e58fff87c --- /dev/null +++ b/.github/workflows/monitor-components.yml @@ -0,0 +1,91 @@ +name: Monitor component updates + +# Git for Windows is a slightly modified subset of MSYS2. Some of its +# components are maintained by Git for Windows, others by MSYS2. To help +# keeping the former up to date, this workflow monitors the Atom/RSS feeds +# and opens new tickets for each new component version. + +on: + schedule: + - cron: "23 8,11,14,17 * * *" + workflow_dispatch: + +env: + CHARACTER_LIMIT: 5000 + MAX_AGE: 7d + +jobs: + job: + # Only run this in Git for Windows' fork + if: github.event.repository.owner.login == 'git-for-windows' + runs-on: ubuntu-latest + permissions: + issues: write + strategy: + matrix: + component: + - label: git + feed: https://github.com/git/git/tags.atom + - label: git-lfs + feed: https://github.com/git-lfs/git-lfs/tags.atom + - label: git-credential-manager + feed: https://github.com/git-ecosystem/git-credential-manager/tags.atom + - label: tig + feed: https://github.com/jonas/tig/tags.atom + - label: cygwin + feed: https://github.com/cygwin/cygwin/releases.atom + title-pattern: ^(?!.*newlib) + - label: msys2-runtime-package + feed: https://github.com/msys2/MSYS2-packages/commits/master/msys2-runtime.atom + - label: msys2-runtime + feed: https://github.com/msys2/msys2-runtime/commits/HEAD.atom + aggregate: true + - label: openssh + feed: https://github.com/openssh/openssh-portable/tags.atom + - label: libfido2 + feed: https://github.com/Yubico/libfido2/tags.atom + - label: libcbor + feed: https://github.com/PJK/libcbor/tags.atom + - label: openssl + feed: https://github.com/openssl/openssl/tags.atom + title-pattern: ^(?!.*alpha) + - label: gnutls + feed: https://gnutls.org/news.atom + - label: heimdal + feed: https://github.com/heimdal/heimdal/tags.atom + - label: git-sizer + feed: https://github.com/github/git-sizer/tags.atom + - label: gitflow + feed: https://github.com/petervanderdoes/gitflow-avh/tags.atom + - label: curl + feed: https://github.com/curl/curl/tags.atom + title-pattern: ^(?!rc-) + - label: mintty + feed: https://github.com/mintty/mintty/releases.atom + - label: 7-zip + feed: https://sourceforge.net/projects/sevenzip/rss?path=/7-Zip + aggregate: true + - label: bash + feed: https://git.savannah.gnu.org/cgit/bash.git/atom/?h=master + aggregate: true + - label: perl + feed: https://github.com/Perl/perl5/tags.atom + title-pattern: ^(?!.*(5\.[0-9]+[13579]|RC)) + - label: pcre2 + feed: https://github.com/PCRE2Project/pcre2/tags.atom + - label: mingw-w64-llvm + feed: https://github.com/msys2/MINGW-packages/commits/master/mingw-w64-llvm.atom + - label: innosetup + feed: https://github.com/jrsoftware/issrc/tags.atom + fail-fast: false + steps: + - uses: git-for-windows/rss-to-issues@v0 + with: + feed: ${{matrix.component.feed}} + prefix: "[New ${{matrix.component.label}} version]" + labels: component-update + github-token: ${{ secrets.GITHUB_TOKEN }} + character-limit: ${{ env.CHARACTER_LIMIT }} + max-age: ${{ env.MAX_AGE }} + aggregate: ${{matrix.component.aggregate}} + title-pattern: ${{matrix.component.title-pattern}} From 3423069c4fbe5ee66d98300950915391d9dfb52a Mon Sep 17 00:00:00 2001 From: Victoria Dye Date: Mon, 4 Apr 2022 15:38:58 -0700 Subject: [PATCH 217/229] fsmonitor: reintroduce core.useBuiltinFSMonitor Reintroduce the 'core.useBuiltinFSMonitor' config setting (originally added in 0a756b2a25 (fsmonitor: config settings are repository-specific, 2021-03-05)) after its removal from the upstream version of FSMonitor. Upstream, the 'core.useBuiltinFSMonitor' setting was rendered obsolete by "overloading" the 'core.fsmonitor' setting to take a boolean value. However, several applications (e.g., 'scalar') utilize the original config setting, so it should be preserved for a deprecation period before complete removal: * if 'core.fsmonitor' is a boolean, the user is correctly using the new config syntax; do not use 'core.useBuiltinFSMonitor'. * if 'core.fsmonitor' is unspecified, use 'core.useBuiltinFSMonitor'. * if 'core.fsmonitor' is a path, override and use the builtin FSMonitor if 'core.useBuiltinFSMonitor' is 'true'; otherwise, use the FSMonitor hook indicated by the path. Additionally, for this deprecation period, advise users to switch to using 'core.fsmonitor' to specify their use of the builtin FSMonitor. Signed-off-by: Victoria Dye --- Documentation/config/advice.adoc | 4 ++++ advice.c | 1 + advice.h | 1 + fsmonitor-settings.c | 34 ++++++++++++++++++++++++++++++-- 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/Documentation/config/advice.adoc b/Documentation/config/advice.adoc index 83039ba48bc797..e36198b46474d1 100644 --- a/Documentation/config/advice.adoc +++ b/Documentation/config/advice.adoc @@ -187,4 +187,8 @@ all advice messages. Shown when the user tries to create a worktree from an invalid reference, to tell the user how to create a new unborn branch instead. + + useCoreFSMonitorConfig:: + Advice shown if the deprecated 'core.useBuiltinFSMonitor' config + setting is in use. -- diff --git a/advice.c b/advice.c index 7253891f92d791..a5c3ba5ca43120 100644 --- a/advice.c +++ b/advice.c @@ -93,6 +93,7 @@ static struct { [ADVICE_SUBMODULE_MERGE_CONFLICT] = { "submoduleMergeConflict" }, [ADVICE_SUGGEST_DETACHING_HEAD] = { "suggestDetachingHead" }, [ADVICE_UPDATE_SPARSE_PATH] = { "updateSparsePath" }, + [ADVICE_USE_CORE_FSMONITOR_CONFIG] = { "useCoreFSMonitorConfig" }, [ADVICE_WAITING_FOR_EDITOR] = { "waitingForEditor" }, [ADVICE_WORKTREE_ADD_ORPHAN] = { "worktreeAddOrphan" }, }; diff --git a/advice.h b/advice.h index a443bc7aebe4e6..1df4642b0aa0a4 100644 --- a/advice.h +++ b/advice.h @@ -60,6 +60,7 @@ enum advice_type { ADVICE_SUBMODULE_MERGE_CONFLICT, ADVICE_SUGGEST_DETACHING_HEAD, ADVICE_UPDATE_SPARSE_PATH, + ADVICE_USE_CORE_FSMONITOR_CONFIG, ADVICE_WAITING_FOR_EDITOR, ADVICE_WORKTREE_ADD_ORPHAN, }; diff --git a/fsmonitor-settings.c b/fsmonitor-settings.c index a6587a8972b184..b4c29f44a27827 100644 --- a/fsmonitor-settings.c +++ b/fsmonitor-settings.c @@ -5,6 +5,7 @@ #include "fsmonitor-ipc.h" #include "fsmonitor-settings.h" #include "fsmonitor-path-utils.h" +#include "advice.h" /* * We keep this structure definition private and have getters @@ -100,6 +101,31 @@ static struct fsmonitor_settings *alloc_settings(void) return s; } +static int check_deprecated_builtin_config(struct repository *r) +{ + int core_use_builtin_fsmonitor = 0; + + /* + * If 'core.useBuiltinFSMonitor' is set, print a deprecation warning + * suggesting the use of 'core.fsmonitor' instead. If the config is + * set to true, set the appropriate mode and return 1 indicating that + * the check resulted the config being set by this (deprecated) setting. + */ + if(!repo_config_get_bool(r, "core.useBuiltinFSMonitor", &core_use_builtin_fsmonitor) && + core_use_builtin_fsmonitor) { + if (!git_env_bool("GIT_SUPPRESS_USEBUILTINFSMONITOR_ADVICE", 0)) { + advise_if_enabled(ADVICE_USE_CORE_FSMONITOR_CONFIG, + _("core.useBuiltinFSMonitor=true is deprecated;" + "please set core.fsmonitor=true instead")); + setenv("GIT_SUPPRESS_USEBUILTINFSMONITOR_ADVICE", "1", 1); + } + fsm_settings__set_ipc(r); + return 1; + } + + return 0; +} + static void lookup_fsmonitor_settings(struct repository *r) { const char *const_str; @@ -126,12 +152,16 @@ static void lookup_fsmonitor_settings(struct repository *r) return; case 1: /* config value was unset */ + if (check_deprecated_builtin_config(r)) + return; + const_str = getenv("GIT_TEST_FSMONITOR"); break; case -1: /* config value set to an arbitrary string */ - if (repo_config_get_pathname(r, "core.fsmonitor", &to_free)) - return; /* should not happen */ + if (check_deprecated_builtin_config(r) || + repo_config_get_pathname(r, "core.fsmonitor", &to_free)) + return; const_str = to_free; break; From 8cf7fcc5025c550d955c25140f4c282a7ae393a6 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 13 Feb 2023 13:31:35 +0100 Subject: [PATCH 218/229] Describe Git for Windows' architecture The Git for Windows project has grown quite complex over the years, certainly much more complex than during the first years where the `msysgit.git` repository was abusing Git for package management purposes and the `git/git` fork was called `4msysgit.git`. Let's describe the status quo in a thorough way. Signed-off-by: Johannes Schindelin --- ARCHITECTURE.md | 116 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 ARCHITECTURE.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000000000..7de4f99bf71ec4 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,116 @@ +# Architecture of Git for Windows + +Git for Windows is a complex project. + +## What _is_ Git for Windows? + +### A fork of `git/git` + +First and foremost, it is a friendly fork of [`git/git`](https://github.com/git/git), aiming to improve Git's Windows support. The [`git-for-windows/git`](https://github.com/git-for-windows/git) repository contains dozens of topics on top of `git/git`, some awaiting to be "upstreamed" (i.e. to be contributed to `git/git`), some still being stabilized, and a few topics are specific to the Git for Windows project and are not intended to be integrated into `git/git` at all. + +### Enhancing and maintaining Git's support for Windows + +On the source code side, Git's Windows support is made a bit more tricky than strictly necessary by the fact that Git does not have any platform abstraction layer (unlike other version control systems, such as Subversion). It relies on the presence of POSIX features such as the `hstrerror()` function, and on platforms lacking that functionality, Git provides shims. That leads to some challenges e.g. with the `stat()` function which is very slow on Windows because it has to collect much more metadata than what e.g. the very quick `GetFileAttributesExW()` Win32 API function provides, even when Git calls `stat()` merely to test for the presence of a file (for which all that gathered metadata is totally irrelevant). + +### Providing more than just source code + +In contrast to the Git project, Git for Windows not only publishes tagged source code versions, but full builds of Git. In fact, Git for Windows' primary purpose, as far as most users are concerned, is to provide a convenient installer that end-users can run to have Git on their computer, without ever having to check out `git-for-windows/git` let alone build it. In essence, Git for Windows has to maintain a separate project altogether in addition to the fork of `git/git`, just to build these release artifacts: [`git-for-windows/build-extra`](https://github.com/git-for-windows/build-extra). This repository also contains the definition for a couple of other release artifacts published by Git for Windows, e.g. the "portable" edition of Git for Windows which is a self-extracting 7-Zip archive that does not need to be installed. + +### A software distribution, really + +Another aspect that contributes to the complexity of Git for Windows is that it is not just building `git.exe` and distributes that. Due to its heritage within the Linux project, Git takes certain things for granted, such as the presence of a Unix shell, or for that matter, a package management system from which dependencies can be fetched and updated independently of Git itself. Things that are distinctly not present in most Windows setups. To accommodate for that, Git for Windows originally relied on the MSys project, a minimal fork of Cygwin providing a Unix shell ("Bash"), a Perl interpreter and similar Unix-like tools, and on the MINGW project, a project to build libraries and executables using a GNU C Compiler that relies only on Win32 API functions. As of Git for Windows v2.x, the project has switched away from [MSys](https://sourceforge.net/projects/mingw/files/MSYS/)/[MinGW](https://osdn.net/projects/mingw/) (due to less-than-active maintenance) to [the MSYS2 project](https://msys2.org). That switch brought along the benefit of a robust package management system based on [Pacman](https://archlinux.org/pacman/) (hailing from Arch Linux). To support Windows users, who are in general unfamiliar with Linux-like package management and the need to update installed packages frequently, Git for Windows bundles a subset of its own fork of MSYS2. To put things in perspective: Git for Windows bundles files from ~170 packages, one of which contains Git, and another one contains Git's help files. In that respect, Git for Windows acts like a distribution more than like a mere single software application. + +Most of MSYS2's packages that are bundled in Git for Windows are consumed directly from MSYS2. Others need forks that are maintained by Git for Windows project, to support Git for Windows better. These forks live in the [`git-for-windows/MSYS2-packages`](https://github.com/git-for-windows/MSYS2-packages) and [`git-for-windows/MINGW-packages`](https://github.com/git-for-windows/MINGW-packages) repositories. There are several reasons justifying these forks. For example, the Git for Windows' flavor of the MSYS2 runtime behaves like Git's test suite expects it while MSYS2's flavor does not. Another example: The Bash executable bundled in Git for Windows is code-signed with the same certificate as `git.exe` to help anti-malware programs get out of the users' way. That is why Git for Windows maintains its own `bash` Pacman package. And since MSYS2 dropped 32-bit support already, Git for Windows has to update the 32-bit Pacman packages itself, which is done in the git-for-windows/MSYS2-packages repository. (Side note: the 32-bit issue is a bit more complicated, actually: MSYS2 _still_ builds _MINGW_ packages targeting i686 processors, but no longer any _MSYS_ packages for said processor architecture, and Git for Windows does not keep all of the 32-bit MSYS packages up to date but instead judiciously decides which packages are vital enough as far as Git is concerned to justify the maintenance cost.) + +### Supporting third-party applications that use Git's functionality + +Since the infrastructure required by Git is non-trivial the installer (or for that matter, the Portable Git) is not exactly light-weight: As of January 2023, both artifacts are over fifty megabytes. This is a problem for third-party applications wishing to bundle a version of Git for Windows, which is often advisable given that applications may depend on features that have been introduced only in recent Git versions and therefore relying on an installed Git for Windows could break things. To help with that, the Git for Windows project also provides MinGit as a release artifact, a zip file that is much smaller than the full installer and that contains only the parts of Git for Windows relevant for third-party applications. It lacks Git GUI, for example, as well as the terminal program MinTTY, or for that matter, the documentation. + +### Supporting `git/git`'s GitHub workflows + +The Git for Windows project is also responsible for keeping the Windows part of `git/git`'s automated builds up and running. On Windows, there is no canonical and easy way to get a build environment necessary to build Git and run its test suite, therefore this is a non-trivial task that comes with its own maintenance cost. Git for Windows provides two GitHub Actions to help with that: [`git-for-windows/setup-git-for-windows-sdk`](https://github.com/git-for-windows/setup-git-for-windows-sdk) to set up a tiny subset of Git for Windows' full SDK (which would require about 500MB to be cloned, as opposed to the ~75MB of that subset) and [`git-for-windows/get-azure-pipelines-artifact`](https://github.com/git-for-windows/get-azure-pipelines-artifact) e.g. to download some regularly pre-built artifacts (for example, when `git/git`'s automated tests ran on an Ubuntu version that did not provide an up to date [Coccinelle](https://coccinelle.gitlabpages.inria.fr/website/) package, this GitHub Action was used to download a pre-built version of that Debian package). + +## Maintaining Git for Windows' components + +Git for Windows uses a combination of [a GitHub App called GitForWindowsHelper](https://github.com/git-for-windows/gfw-helper-github-app) (to listen for so-called [slash commands](https://github.com/git-for-windows/gfw-helper-github-app#slash-commands)) combined with workflows in [the `git-for-windows-automation` repository](https://github.com/git-for-windows/git-for-windows-automation/) (for computationally heavy tasks) to support Git for Windows' repetitive tasks. + +This heavy automation serves two purposes: + +1. Document the knowledge about "how things are done" in the Git for Windows project. +2. Make Git for Windows' maintenance less tedious by off-loading as many tasks onto machines as possible. + +One neat trick of some `git-for-windows-automation` workflows is that they "mirror back" check runs to the targeted PRs in another repository. This essentially allows versioning the source code independently of the workflow definition. + +Here is a diagram showing how the bits and pieces fit together. + +```mermaid +graph LR + A[`monitor-components`] --> |opens| B + B{issues labeled
`component-update`} --> |/open pr| C + C((GitForWindowsHelper)) --> |triggers| D + D[`open-pr`] --> |opens| E + E{PR in
MINGW-packages
MSYS2-packages
build-extra} --> |closes| B + E --> |/deploy| F + F((GitForWindowsHelper)) --> |triggers| G + G[`build-and-deploy`] --> |deploys to| H + H{Pacman repository} + C --> |backed by| I + F --> |backed by| I + I[[Azure Function]] + D --> |running in| J + G --> | running in| J + J[[git-for-windows-automation]] + K[[git-sdk-32
git-sdk-64
git-sdk-arm64]] --> |syncing from| H + B --> |/add release note| L + L[`add-release-note`] +``` + +For the curious mind, here are [detailed instructions how the Azure Function backing the GitForWindowsHelper GitHub App was set up](https://github.com/git-for-windows/gfw-helper-github-app#how-this-github-app-was-set-up). + +### The `monitor-components` workflow + +When new versions of components that Git for Windows builds become available, new Pacman packages have to be built. To this end, [the `monitor-components` workflow](https://github.com/git-for-windows/git/blob/main/.github/workflows/monitor-components.yml) monitors a couple of RSS feeds and opens new tickets labeled `component-update` for such new versions. + +### Opening Pull Requests to update Git for Windows' components + +After determining that such a ticket indeed indicates the need for a new Pacman package build, a Git for Windows maintainer issues the `/open pr` command via an issue comment ([example](https://github.com/git-for-windows/git/issues/4281#issuecomment-1426859787)), which gets picked up by the GitForWindowsHelper GitHub App, which in turn triggers [the `open-pr` workflow](https://github.com/git-for-windows/git-for-windows-automation/blob/main/.github/workflows/open-pr.yml) in the `git-for-windows-automation` repository. + +### Deploying the Pacman packages + +This will open a Pull Request in one of Git for Windows' repositories, and once the PR build passes, a Git for Windows maintainer issues the `/deploy` command ([example](https://github.com/git-for-windows/MINGW-packages/pull/69#issuecomment-1427591890)), which gets picked up by the GitForWindowsHelper GitHub App, which triggers [the `build-and-deploy` workflow](https://github.com/git-for-windows/git-for-windows-automation/blob/main/.github/workflows/build-and-deploy.yml). + +### Adding release notes + +Finally, once the packages have been built and deployed to the Pacman repository (which is hosted in Azure Blob Storage), a Git for Windows maintainer will merge the PR(s), which in turn will close the ticket, and the maintainer then issues an `/add release note` command ([example](https://github.com/git-for-windows/MINGW-packages/pull/69#issuecomment-1427782230)), which again gets picked up by the GitForWindowsHelper GitHub App that triggers [the `add-release-note` workflow](https://github.com/git-for-windows/build-extra/blob/main/.github/workflows/add-release-note.yml) that creates and pushes a new commit to the `ReleaseNotes.md` file in `build-extra` ([example](https://github.com/git-for-windows/build-extra/commit/b39c148ff8dc0e987afdb677d17c46a8e99fd0ef)). + +## Releasing official Git for Windows versions + +A relatively infrequent part of Git for Windows' maintainers' duties, if the most rewarding part, is the task of releasing new versions of Git for Windows. + +Most commonly, this is done in response to the "upstream" Git project releasing a new version. When that happens, a Git for Windows maintainer runs [the helper script](https://github.com/git-for-windows/build-extra/blob/main/shears.sh) to perform a "merging rebase" (i.e. a rebase that starts with a fake-merge of the previous tip commit, to maintain both a clean set of commits as well as a [fast-forwarding](https://git-scm.com/docs/git-merge#Documentation/git-merge.txt---ff-only) commit history). + +Once that is done, the maintainer will open a Pull Request to benefit from the automated builds and tests ([example](https://github.com/git-for-windows/git/pull/4160)) as well as from reviews of the [`range-diff`](https://git-scm.com/docs/git-range-diff) relative to the current `main` branch. + +Once everything looks good, the maintainer will issue the `/git-artifacts` command ([example](https://github.com/git-for-windows/git/pull/4160#issuecomment-1346801735)). This will trigger an automated workflow that builds all of the release artifacts: installers, Portable Git, MinGit, `.tar.xz` archive and a NuGet package. Apart from the NuGet package, two sets of artifacts are built: targeting 32-bit ("x86") and 64-bit ("amd64"). + +Once these artifacts are built, the maintainer will download the installer and run [the "pre-flight checklist"](https://github.com/git-for-windows/build-extra/blob/main/installer/checklist.txt). + +If everything looks good, a `/release` command will be issued, which triggers yet another workflow that will download the just-built-and-verified release artifacts, publish them as a new GitHub release, publish the NuGet packages, deploy the Pacman packages to the Pacman repository, send out an announcement mail, and update the respective repositories including [Git for Windows' website](https://gitforwindows.org/). + +As mentioned [before](#architecture-of-git-for-windows), the `/git-artifacts` and `/release` commands are picked up by the GitForWindowsHelper GitHub App which subsequently triggers the respective workflows in the `git-for-windows-automation` repository. Here is a diagram: + +```mermaid +graph LR + A{Pull Request
updating to
new Git version} --> |/git-artifacts| B + B((GitForWindowsHelper)) --> |triggers| C + C[`tag-git`] --> |upon successful build
triggers| D + D((GitForWindowsHelper)) --> |triggers| E + E[`git-artifacts`] + E --> |maintainer verifies artifacts| E + A --> |upon verified `git-artifacts`
/release| F + F[`release-git`] + C --> |running in| J + E --> | running in| J + F --> | running in| J + J[[git-for-windows-automation]] +``` \ No newline at end of file From b3629b3f996acbc2c91fd0f69e1e38e8caccc84b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 26 Jan 2026 19:18:40 +0100 Subject: [PATCH 219/229] Add an AGENTS.md file to help with AI-assisted debugging/development MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In this time and age, AI is everywhere. However, it's sometimes not very easy to use. For green-field projects it works quite a bit better than for existing legacy projects. And Git's source code is _quite_ as legacy code as they come... 😁 Now, the only way how AI can be used efficiently with legacy code is by providing enough information by way of prompt context for the AI to have a chance to make any sense of the code. The structure and the architecture is, after all, not designed for AI, but rather the opposite: By virtue of having grown organically over two decades, there is no design that AI coding models would readily grasp. So here is a document that describes all kinds of aspects about this project. The idea is to help AI by providing information that it does not have ingrained in its weights. The idea is to provide information that a human prompter might take for granted, but no coding model will have been trained on specifically. Assisted-by: Claude Opus 4.5 Signed-off-by: Johannes Schindelin --- .gitattributes | 1 + AGENTS.md | 1225 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1226 insertions(+) create mode 100644 AGENTS.md diff --git a/.gitattributes b/.gitattributes index 9d6e484d1ec0c4..9225e8a16dcc82 100644 --- a/.gitattributes +++ b/.gitattributes @@ -7,6 +7,7 @@ *.py text eol=lf diff=python *.bat text eol=crlf *.png binary +/AGENTS.md conflict-marker-size=32 CODE_OF_CONDUCT.md -whitespace /Documentation/**/*.adoc text eol=lf whitespace=trail,space,incomplete /command-list.txt text eol=lf diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 00000000000000..7273095017731a --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,1225 @@ +# Git for Windows - Development Guide + +## Background + +Git for Windows is a fork of upstream Git that provides the necessary +adaptations to make Git work well on Windows. While the primary target is +Windows, the project also maintains working builds on other platforms (Linux, +macOS) because cross-platform builds often catch mistakes that might be missed +when testing only on Windows. + +There are downstream projects that build on Git for Windows, such as Microsoft +Git, which adds features for large monorepos hosted on Azure DevOps. + +## Overview + +This document provides guidance for developing and debugging in +Git for Windows. + +## Repository Structure + +### Branch Naming Patterns + +Based on actual repository usage: + +- `main` - The primary development branch +- Feature branches use descriptive topic names, targeting the main branch + +## Building and Testing + +### Build + +```bash +make -j$(nproc) +``` + +On Windows (in a Git for Windows SDK shell): + +```bash +make -j15 +``` + +When driving the build non-interactively (for example from PowerShell, or +from an automation agent, rather than an interactive SDK shell), do not use +a login shell. A login shell (`bash -l` / `bash --login`) re-runs the +profile scripts and is unnecessary once `MSYSTEM` and `PATH` are set +explicitly. Instead set `MSYSTEM` and prepend the SDK's binary directories +to `PATH`, then invoke a non-login `bash -c` (replace `C:\git-sdk-64` with +your SDK root): + +```powershell +$env:MSYSTEM = "MINGW64" +$env:PATH = "C:\git-sdk-64\mingw64\bin;C:\git-sdk-64\usr\bin;" + $env:PATH +& C:\git-sdk-64\usr\bin\bash.exe -c "make -j15" +``` + +If the link step fails to find `target/release/libgitcore.a` (the optional +Rust component), pass `NO_RUST=1` to skip the cargo step: + +```powershell +& C:\git-sdk-64\usr\bin\bash.exe -c "make -j15 NO_RUST=1" +``` + +### Run Specific Tests + +```bash +cd t && sh t0001-init.sh # Run normally +cd t && sh t0001-init.sh -v # Verbose +cd t && sh t0001-init.sh -ivx # verbose, trace, fail-fast +``` + +Some tests are expensive and skipped by default. When a test exits immediately +with "skip all", check the test script header for `test_bool_env GIT_TEST_*` +to find which environment variable enables it. + +## Git Source Code Structure + +This section provides a bird's eye view of Git's source code layout. For +more details, see "A birds-eye view of Git's source code" in +`Documentation/user-manual.adoc`. + +### Key Directories + +| Directory | Purpose | +|------------------|----------------------------------------------------| +| `builtin/` | Built-in command implementations (`cmd_()`) | +| `xdiff/` | Low-level diff algorithms (libxdiff) | +| `t/` | Test suite (shell scripts, helpers, libraries) | +| `Documentation/` | Man pages, guides, technical docs (AsciiDoc) | +| `contrib/` | Optional extras, not part of core Git | +| `compat/` | Platform compatibility shims | +| `refs/` | Reference backends (files, reftable) | +| `reftable/` | Reftable format implementation | + +### Built-in Commands + +Built-in commands are implemented in `builtin/.c` with a function +`cmd_()`. To add a new built-in: + +1. Create `builtin/.c` implementing `cmd_()` +2. Add entry to the `commands[]` array in `git.c`: + ```c + { "", cmd_, RUN_SETUP }, + ``` +3. Add to `BUILTIN_OBJS` in `Makefile` +4. Add to `command-list.txt` with appropriate category +5. Run `make check-builtins` to verify consistency + +### Object Data Model + +Git stores four types of objects, defined in `object.h`: + +```c +enum object_type { + OBJ_COMMIT = 1, /* Points to tree, has parent commits, metadata */ + OBJ_TREE = 2, /* Directory listing: names -> blob/tree OIDs */ + OBJ_BLOB = 3, /* File contents */ + OBJ_TAG = 4, /* Annotated tag pointing to another object */ +}; +``` + +Objects are addressed by their SHA (OID) and stored in the Object Database. + +### Object Database (ODB) + +The ODB is defined in `odb.h` and implemented in `odb.c`: + +- **`struct object_database`**: Top-level container, owned by a repository + - `sources`: Linked list of `odb_source` (primary + alternates) + - `replace_map`: Object replacements (see `git-replace(1)`) + - `commit_graph`: Commit-graph cache for faster traversal + +- **`struct odb_source`**: A single object store location + - `path`: Directory (e.g., `.git/objects` or an alternate) + - `loose`: Loose object cache + - `packfiles`: Packfile store (idx + pack files) + +Key functions: +- `odb_read_object()`: Read an object by OID +- `odb_write_object()`: Write an object, returns OID +- `odb_read_object_info()`: Get object type/size without reading content + +### Documentation + +Documentation lives in `Documentation/` as AsciiDoc (`.adoc`) files: + +- `git-.adoc` - Man pages for commands +- `config/.adoc` - Config option documentation (included by others) +- `technical/` - Technical specifications and internals + +To build documentation: +```bash +make -C Documentation html # Build HTML docs +make -C Documentation man # Build man pages +``` + +To add documentation for a new config option, add it to the appropriate +file in `Documentation/config/`. These are included by other docs. + +To lint documentation: +```bash +make -C Documentation lint-docs +``` + +## Debugging Techniques + +### Debugging Philosophy + +Debugging is not about guessing fixes and seeing if they work. It is about +building a complete understanding of the problem before attempting any fix. +The goal is not speed to a "fix" but confidence that you understand and have +addressed the root cause. + +**Respect turnaround time.** If seeing the result of an attempted fix takes +7-10 minutes (e.g., a CI workflow run), you cannot afford to guess. Each +iteration costs human time and attention. Before pushing any change: + +1. Ask: "What information am I missing to competently assess this situation?" +2. Add diagnostic output that will provide that information if the fix fails. +3. Consider whether you can reproduce the issue locally where turnaround is + seconds, not minutes. + +**Understand before acting.** Before attempting any fix: + +1. When investigating a regression between two versions, start by examining + the code diff. Analyze what actually changed before running any tests. + Tests confirm hypotheses; reading the diff gives you the hypothesis. +2. Trace the code flow completely. Read the relevant Makefiles, scripts, and + source files. Understand what each component does and how they interact. +3. Identify all changes that could have contributed: upstream commits, + downstream patches, infrastructure changes (CI runner updates, dependency + upgrades). +4. For each potential cause, find the specific commit, its date, its intent, + and how it interacts with other components. +5. Build a hypothesis. Then ask: "How would I confirm or disprove this?" + +**Do not assume root cause from symptoms.** A symptom appearing on one +platform does not mean the bug is platform-specific. The cause may be in +shared code that manifests differently across platforms. Similarly, a passing +test on one platform when it fails on another is data to investigate, not +grounds to conclude "works for me." + +**When a fix does not work, investigate why.** If you expected a fix to work +and it did not, that is valuable information. Do not abandon that line of +thinking and try something else. Instead: + +1. Ask: "Why didn't that work? What does this tell me about my understanding?" +2. Add more targeted diagnostics to understand the discrepancy. +3. Re-examine your assumptions. Something you believed to be true is false. + +**Add diagnostics proactively.** Before pushing a fix attempt, add diagnostic +output that will: + +1. Confirm the state you expect to see if the fix works. +2. Reveal the actual state if it does not. +3. Provide enough context to understand the next step without another round + trip. + +For build failures, this might include: library paths, compiler flags, +architecture information, symbol tables, file existence checks, environment +variables. + +**Build confidence before pushing.** A fix should not be a guess. You should +be able to explain: + +1. What was the root cause? +2. Why does this fix address it? +3. What other ways could this problem be solved? +4. Am I choosing the "most correct" or "most effective" approach? +5. What evidence confirms your understanding? +6. What could still go wrong, and how would you detect it? + +### Searching the Codebase + +In particular when debugging failures that printed error messages, it is often +a useful thing to search for those error messages; If parts of the message seem +mutable (e.g. commit OIDs), those will not be hard-coded and the search needs +to accommodate for that by using regular expressions or prefix matches. + +Use `git grep` for fast code searches: + +```bash +git grep -n -i "pattern" # Case-insensitive search with line numbers +git grep -n -w "word" # Whole-word matches only +git grep -n -i "pattern" -- "*.c" # Search only C files +``` + +### Trace2 + +Enable tracing to see command execution patterns: +```bash +GIT_TRACE2_EVENT=/path/to/trace.txt git +``` + +### Instrumenting Git Internals During Tests + +When adding debug output to Git's C code during test investigation, +`fprintf(stderr, ...)` from git subprocesses spawned by the test framework +is typically swallowed (redirected or discarded by the test harness). Use +Trace2 instead: + +```c +trace2_data_intmax("index", NULL, "my_debug/cache_nr", istate->cache_nr); +trace2_data_string("index", NULL, "my_debug/state", some_string); +``` + +Then run the test with `GIT_TRACE2_EVENT` or `GIT_TRACE2_PERF` pointing to +a file, and grep the output. This integrates with Git's existing tracing +infrastructure and survives the test framework's output management. + +As a last resort (e.g. when Trace2 is not initialized yet at the point you +need to instrument), write to a fixed file path: + +```c +FILE *f = fopen("/tmp/debug.log", "a"); +if (f) { fprintf(f, "state: %u\n", value); fclose(f); } +``` + +### Comparing Branches After Rebase + +```bash +# See what patches exist in a new branch but not old +git log --oneline old-branch..new-branch +# or +git range-diff -s --right-only old-branch...new-branch + +# Compare specific files between branches +git diff old-branch..new-branch -- path/to/file.c +# or +git log -p old-branch..new-branch -- path/to/file.c +# or even +git log -L start-line,end-line:path/to/file.c old-branch..new-branch -- + +# Find upstream changes between tags +git log --oneline --first-parent v2.52.0..v2.53.0 +``` + +### Test Failure Investigation + +1. **Reproduce with tracing**: Run test with `-ivx` flags +2. **Check timestamps**: Look at `t_abs` in trace to understand ordering +3. **Compare with working version**: Build and test the previous version +4. **Bisect if needed**: Use `git bisect` to find the breaking commit + +Bisecting failures introduced by upstream commits require some stunts to +apply the downstream changes for every bisection step. This can be done by +squashing all downstream changes into one throw-away commit and then +cherry-picking that (typically, there will be merge conflicts the farther +away from the original branch point the commit is cherry-picked to, so it +often makes sense to squash both old and new downstream changes, and then +to "interpolate" between them when encountering merge conflicts). + +### Bisecting Failures in `seen` + +When a topic passes on its own but fails after being merged to `seen`, the +failure is caused by interaction with another in-flight topic. To identify +the culprit: + +1. Fetch the exact `seen` commit from the failing CI run (get the SHA from + the workflow run metadata via the GitHub API). +2. Use a worktree checked out at that `seen` commit. +3. Bisect the first-parent history between `upstream/master` and `seen~1` + (excluding the topic's own merge). At each bisection step, merge the + topic in temporarily, build, run the test, then undo the merge. +4. Write a `git bisect run` script that automates this. Key pitfalls: + - The script must `unset` test environment variables (especially + `GIT_TEST_SPLIT_INDEX`) before cleanup operations like + `git checkout -f`, otherwise the worktree's own index can get + corrupted. + - Use `git checkout -f "$ORIG"` (not `git reset --hard`) to undo the + temporary merge, since `reset --hard` under split-index can corrupt. + - Save the current commit OID at the start (`ORIG=$(git rev-parse HEAD)`) + because `ORIG_HEAD` is unreliable during bisect. + - On merge conflict, return 125 (skip) and `git merge --abort`. +5. Store the alias for running with the full set of CI test variables as a + repository-local alias (to avoid repeating the long export list and to + allow the user to approve the tool call once). + +### CI/Workflow Failure Investigation + +When a CI workflow fails, the debugging process has a high cost per iteration. +Approach these failures methodically: + +**1. Establish what changed.** Before looking at the error, identify: + +- What was the last successful run? What version/commit was it based on? +- What changed between then and now? (upstream commits, downstream patches, + runner image updates, dependency changes) +- Use the GitHub API to retrieve run metadata and compare. + +**2. Analyze the error deeply.** Read the full error message and surrounding +context. Understand: + +- What command failed? +- What were its inputs (flags, environment, paths)? +- What did it expect vs. what did it get? + +**3. Trace the code flow locally.** Before making any CI changes: + +- Read the workflow YAML, Makefiles, and scripts involved. +- Understand how variables flow from one to another. +- Identify where the failing values come from. + +**4. Reproduce locally if possible.** Many CI failures can be reproduced +locally with faster turnaround: + +- For build failures: replicate the build environment and commands. +- For macOS issues: if you lack a Mac, at least trace the Makefile logic + to understand what flags should be set and why. +- For test failures that only appear in specific CI jobs (like + `linux-TEST-vars`): reproduce with the _exact_ set of environment + variables that job sets. Check `ci/run-build-and-tests.sh` for the + job's variable block. Do not assume a single variable (e.g. + `GIT_TEST_SPLIT_INDEX`) is sufficient; other variables may contribute + to the failure path. +- When a test fails in `seen` but not on the topic branch alone, check + out the exact `seen` commit from the failing CI run (get the SHA from + the workflow run metadata) and reproduce against that. The interaction + with other in-flight topics is the likely cause. + +**5. Do not assume CI coverage from platform support.** When asking "why +does platform X not see this bug?", verify whether CI actually tests that +combination on that platform. For example, `GIT_TEST_SPLIT_INDEX=yes` is +only set by `linux-TEST-vars`; there is no equivalent `osx-TEST-vars` or +`windows-TEST-vars` job. A bug that only manifests under split-index +testing may be present on all platforms but only caught on Linux. + +**5. Add comprehensive diagnostics on first attempt.** If you must push to +CI to test, make that push count: + +- Add diagnostic output for every hypothesis you have. +- Print the values of key variables, paths, flags. +- Show the state before and after key operations. +- Design diagnostics to distinguish between your hypotheses. + +**6. Do not remove diagnostics until the problem is solved.** Keep them in +"drop!" commits so they can be easily removed later but provide information +if subsequent fixes also fail. + +**7. When a fix fails, treat it as data.** The failure tells you something. +Your mental model was wrong. Figure out what before trying again. + +## Git Workflow + +This repository is a shared development environment, not a sandbox. Exercise +caution with all Git operations. + +### Committing Changes + +Never use `git add -A` or `git add .` - these commands will stage untracked +build artifacts, editor swap files, and other detritus that should not be +committed. Always specify pathspecs explicitly: + +```bash +# Good: stage and commit specific files +git commit -sm "your message here" path/to/file.c other/file.h + +# Bad: stages everything, including untracked garbage +git add -A && git commit -m "message" +``` + +The `-s` flag adds a Signed-off-by trailer, which is required for this +project. + +When AI assistance is used to author or co-author a commit, add a +Co-authored-by trailer identifying the model: + +```bash +git commit -s --trailer "Co-authored-by: " -m "message" file.c +``` + +### Pushing Changes + +Never push without explicit user permission. The user controls when and +where changes are pushed. This is especially critical because: + +- The repository has multiple remotes with different purposes +- Force-pushing to the wrong remote can cause significant damage +- Tags require special handling (`git push --tags` or explicit tag pushes) + +Wait for the user to push, or ask explicitly before pushing. + +### Making Code Changes + +**Minimal, surgical changes.** Make the smallest possible change to achieve +the goal. Do not rewrite entire files or functions when a targeted edit +suffices. When removing functionality: + +1. Remove the code paths that invoke the unwanted functionality +2. Compile to identify what is now unused +3. Remove the unused functions one at a time +4. Repeat until clean + +**No fly-by changes.** Do not make changes that were not requested, even if +they seem like improvements (renaming variables, reformatting untouched code, +"fixing" things not part of the task). If you believe a change would be +beneficial but it was not requested, ask for permission first. + +**The human is the driver.** Execute what is asked. If you think something +should be done differently, ask---do not just do it. + +### Commit Message Quality + +Good commit messages use flowing English prose, not bullet points. They +clearly state: + +- **Context**: What situation prompted this change? Include URLs to failing + CI runs, issue numbers, or other references that future readers will need. +- **Intent**: What is this change trying to accomplish? +- **Justification**: Why is this the right approach? What alternatives were + considered? When choosing between approaches based on performance, + include measured timings so future readers understand the tradeoffs. +- **Implementation**: How does the change work? (Only for non-obvious parts; + don't describe what's clear from the diff.) + +Include exact error messages rather than vague descriptions. If a build +failed with `Undefined symbols for architecture arm64: "_iconv"`, put that +in the commit message - don't just say "fixed a linker error." + +Wrap commit messages at 76 columns per line. + +### Commit Prefixes for Rebase Workflows + +This repository uses interactive rebase with autosquash. Commit prefixes +signal intent: + +- **`fixup! `**: Will be squashed into the referenced commit + during rebase. The title after `fixup!` must match the original commit's + title exactly. +- **`drop!`**: Indicates a commit that should be dropped before the final + merge. Used for debugging, temporary workarounds, or experiments. + +To find the correct title for a fixup commit: + +```bash +git log --oneline path/to/changed/file | head -10 +``` + +Then use the exact title: + +```bash +git commit -sm "fixup! release: add Mac OSX installer build" path/to/file +``` + +## Rebasing Workflow + +Rebases are the bread and butter of Git for Windows: topic branches are +rebased every time upstream Git releases a new version. This section covers +the workflow for managing downstream patches through repeated rebases. + +### Merging-Rebases + +Git for Windows uses "merging-rebases" to maintain downstream patches. Unlike +a flat series of commits, the downstream changes are organized as topic +branches merged together, preserving the logical grouping of related changes. + +Each integration branch (`main`, `shears/next`, `shears/seen`) contains a +marker commit with the message "Start the merging-rebase to \". This +commit separates upstream history from downstream patches. Reference it with: + +```bash +# Find the marker commit +git log --oneline --grep="Start the merging-rebase" -1 + +# Reference it using commit message search syntax +origin/main^{/Start.the.merging-rebase} +``` + +When working with merging-rebases: + +- **Downstream patches start after the marker**: Use + `origin/main^{/Start.the.merging-rebase}..origin/main` to see all + downstream commits +- **Topic branches are merged, not rebased flat**: Each logical feature or + fix is a branch merged into the integration branch +- **Merge commits are preserved**: The rebase recreates the merge structure + on top of the new upstream base + +To compare downstream patches before and after a rebase: + +```bash +# Compare the old and new downstream patch series +git range-diff \ + old-base^{/Start.the.merging-rebase}..old-branch \ + new-base^{/Start.the.merging-rebase}..new-branch +``` + +### Starting a Merging-Rebase + +To rebase the downstream patches onto a new upstream version, create a marker +commit and use it as the base for an interactive rebase: + +```bash +# Variables for the commit message +tag=v2.53.0 +# The previous marker - this becomes the exclusion point for --onto +previousMergeOid=$(git rev-parse origin/main^{/Start.the.merging-rebase}) +tagOid=$(git rev-parse "$tag") +tipOid=$(git rev-parse origin/main) + +# Create the marker commit with two parents: the tag and the current tip +markerOid=$(git commit-tree "$tag^{tree}" -p "$tag" -p "$tipOid" -m "Start the merging-rebase to $tag + +This commit starts the rebase of $previousMergeOid to $tagOid") + +# Graft the marker to appear as if it has only the tag as parent +git replace --graft "$markerOid" "$tag" + +# Use the marker as the base for rebasing (only commits after previousMergeOid) +git rebase -r --onto "$markerOid" "$previousMergeOid" origin/main + +# After the rebase completes, delete the replace ref +git replace -d "$markerOid" +``` + +The marker commit is created with two parents: the upstream tag and the +current branch tip. The `git replace --graft` makes Git see only the tag as +parent during the rebase, allowing the downstream commits to be cleanly +rebased onto the new upstream. After the rebase completes, the replace ref +is deleted to clean up. + +#### The shears/* Branches + +Upstream Git has four integration branches: `seen`, `next`, `master`, and +`maint`. Git for Windows maintains a corresponding `shears/*` branch for each +(`shears/seen`, `shears/next`, `shears/master`, `shears/maint`) that +continuously rebases Git for Windows' `main` onto the respective upstream +branch. + +These branches are updated incrementally rather than from scratch, avoiding +re-resolution of merge conflicts. The update process leverages reachability: + +1. **Integrate new downstream commits**: If `origin/main` has commits not yet + in the shears branch, rebase them on top (using `-r` to preserve branch + structure). Update the marker commit's message and second parent. + +2. **Integrate new upstream commits**: If the upstream branch has commits not + yet integrated, rebase onto the new upstream tip. Update the marker commit + accordingly. + +The marker commit's second parent always points to the current `origin/main` +tip, making it trivial to identify what downstream commits are included. +Similarly, the marker's first parent (the upstream base) shows exactly which +upstream version is integrated. + +### When to Skip a Patch + +Use `git rebase --skip` when the patch is already in the new base: + +- **Upstreamed**: The patch was accepted upstream and is now in `seen` +- **Backported**: A fix we backported is now included in the upstream base +- **Superseded**: HEAD already contains evolved code that includes this + change + +Signs to skip rather than resolve: HEAD has the functionality, the +conflict would discard the patch entirely, or `git range-diff` shows +the downstream and upstream patches are equivalent. + +To find the corresponding upstream commit for a conflicting patch: + +```bash +git range-diff --left-only REBASE_HEAD^! REBASE_HEAD.. +``` + +### Resolving Merge Conflicts + +When resolving merge conflicts during a rebase (especially when squashing +fixups), the goal is to **apply the minimal surgical change** that the +patch intended, not to reconstruct entire functions or add duplicate code. + +#### 1. Understand What the Patch Wants + +First, examine the patch being applied: + +```bash +git show REBASE_HEAD +``` + +Look at the actual changes (lines starting with `-` and `+`): +- What lines are being removed? +- What lines are being added? +- What is the context (function name, nearby code)? + +**Key insight**: The patch shows the *intent*---a specific small change to +make. Focus on this, not on the conflict markers' content. + +**Code movement detection**: If the patch shows large changes, check with +`--ignore-space-change`: + +```bash +git show --ignore-space-change +``` + +This reveals whether the commit is primarily **moving code** (lots of +whitespace changes) or making **logic changes** (actual code modifications). +When code was moved and re-indented, focus only on the non-whitespace +changes when resolving the conflict. + +#### 2. Understand Where the Code Is Now + +The conflict occurred because the code moved or changed since the patch was +created. Find where that code actually exists now: + +```bash +# If the patch was changing a specific pattern, find all occurrences +git grep -n "pattern from patch" + +# View the conflicted file around those locations +``` + +**Common mistake**: Assuming the conflict markers show you what to do. They +do not---they just show where Git got confused. + +#### 3. Apply the Surgical Change + +Make **only** the change the patch intended, but in the current location: + +- If the patch adds `--abbrev=12` to a range-diff call, find where that + range-diff call is NOW and add it there +- If the patch changes a `.split()` pattern, find where that pattern is NOW + and change it +- Do not copy entire functions from the conflict markers +- Do not create duplicates + +#### 4. Remove ALL Conflict Markers + +Conflict markers make the file invalid code: +``` +<<<<<<< HEAD +======= +>>>>>>> commit-hash +``` + +**All three types of markers must be completely removed.** + +#### 5. Verify the Resolution + +**Critical**: After staging your resolution, verify it matches the patch +intent: + +```bash +# Compare your staged changes to the original patch +git diff --cached +git rebase --show-current-patch + +# Or more directly, compare to REBASE_HEAD +git diff --cached +git show REBASE_HEAD + +# For code that was moved/re-indented, ignore whitespace +git diff --cached --ignore-space-change +git show REBASE_HEAD --ignore-space-change +``` + +**Verify, verify, verify**: The output of `git diff --cached` should +correspond closely to the diff in `git show REBASE_HEAD`. The line numbers +and context will differ (because code moved), but the actual changes (the +`-` and `+` lines) should match the patch intent. + +**After completing a rebase**, always verify the final result: + +```bash +# Compare tree before and after rebase +git diff @{1} + +# Shows what changed in each rebased commit +git range-diff @{1}... +``` + +If the rebase was onto the same base commit (e.g., squashing fixups), the +`git diff @{1}` should be empty---this proves the rebase only reorganized +commits without changing the end result. If the rebase was onto a new base +commit (e.g., rebasing onto a new upstream release), the diff should match +the difference between the old and new base commits, modulo any changes +from upstreamed or backported patches. The `git range-diff @{1}...` shows +the intended amendments (like adding `--abbrev=12`) were correctly applied +to each commit. + +### Conflict Resolution Red Flags + +These indicate you are doing it wrong: + +- Your diff adds hundreds of lines when the patch only changed 3 +- Conflict markers remain in the file +- Functions appear twice in the file +- You added `<<<<<<< HEAD` or `=======` to the staged changes +- Syntax check fails after resolution + +### Key Conflict Resolution Lessons + +1. **Context changes, intent does not** - The patch's line numbers are + wrong, but the change is right +2. **Conflict markers lie** - They show you where Git got confused, not + what you should do +3. **One change at a time** - If the patch adds one line, your resolution + should add one line +4. **Verify, verify, verify** - `git diff --cached` should match + `git show REBASE_HEAD` (modulo context) +5. **Post-rebase verification** - `git diff @{1}` (empty) and + `git range-diff @{1}...` (shows amendments) +6. **Ignore whitespace for code moves** - Use `--ignore-space-change` to + see the actual logic changes when code was moved and re-indented +7. **When in doubt, look at the range-diff** - `git range-diff` shows if + you matched the intent + +### Useful Rebase Tools + +- `git rebase --show-current-patch` - See what change is being applied +- `git show REBASE_HEAD` - Alternative to above, works better with + `--ignore-space-change` +- `git show --ignore-space-change` - See only logic changes, not + whitespace/indentation +- `git grep -n "pattern"` - Find where code moved to +- `git log -L ,: REBASE_HEAD..HEAD` - See how upstream + modified a line range since the original patch; invaluable for + understanding how conflicting lines changed +- `git diff --cached` - After staging resolution, verify it matches + REBASE_HEAD +- `git diff @{1}` - After rebase, compare tree before/after +- `git range-diff @{1}...` - After rebase, verify intended changes were made +- `git range-diff A^! B^!` - Compare original patch to your resolution + +### Leveraging Rerere + +Git's "reuse recorded resolution" (`rerere`) feature automatically records +how you resolve conflicts and replays those resolutions when the same +conflict recurs. This is invaluable for repeated rebases where the same +downstream patches conflict with similar upstream changes. + +When you see `Staged 'file' using previous resolution`, Git has applied a +previously recorded resolution. Always verify these auto-resolutions are +still correct---upstream context may have changed enough that the old +resolution no longer applies cleanly. + +To enable rerere: +```bash +git config --global rerere.enabled true +``` + +### Automation Tips + +When running rebases in automated or scripted contexts, disable the pager +to avoid hangs: + +```bash +GIT_PAGER=cat git range-diff ... +# or +git --no-pager log ... +``` + +### Non-interactive "Interactive" Rebases + +AI agents cannot drive interactive editors reliably. Instead, insert a +`break` as the first todo command so the rebase stops immediately, then +edit the todo file directly: + +```bash +# Start the rebase, stopping before any picks execute +GIT_SEQUENCE_EDITOR='sed -i 1ib' git rebase -ir + +# Find and edit the todo file with the view/edit tools +git rev-parse --git-path rebase-merge/git-rebase-todo + +# After editing the todo, continue (GIT_EDITOR=true suppresses the +# editor that fixup -C and amend! commands would otherwise open) +GIT_EDITOR=true git rebase --continue +``` + +### Scripted Hunk Staging + +`git add -p` is interactive by default, but its prompts follow a +predictable protocol. To stage the first hunk of a file without +human interaction: + +```bash +printf '%s\n' s y q | git add -p +``` + +The `s` splits a large hunk, `y` stages the first sub-hunk, and `q` +quits. Adjust the sequence for different hunk selections (e.g., +`y y n q` to stage the first two hunks but skip the third). + +### Finding Which Commit to Amend + +When a working-tree change belongs in an earlier commit (an `hg absorb` +workflow), use `git log -L` to find which commit last touched the +relevant lines: + +```bash +git log -L ,+: +``` + +This shows the full history of a line range, making it easy to identify +the commit whose title you need for a `fixup!` commit. This is far more +surgical than grepping through full diffs. + +### Fixup Commits + +Downstream patches sometimes require adjustment due to changes in the +environment they operate in. These changes may come from: + +- **Upstream code changes**: API modifications, struct field moves, + declarations relocating between headers, or semantic changes in functions + that downstream code depends on. +- **External environment changes**: CI runner image updates, toolchain + upgrades, dependency version changes, or platform behavior shifts. + +In both cases, create a `fixup!` commit that will be squashed into the +original downstream patch during the next interactive rebase. The commit +message body must precisely document the change that necessitated the fix: + +- For upstream changes: reference the specific upstream commit (by OID or + title) and explain what it changed. +- For external changes: include URLs to failing CI runs, document what + changed in the environment (e.g., "GitHub Actions macos-latest runner + upgraded from macOS 14 to macOS 15"), and note the exact error message. + +This documentation is essential because the fixup will be squashed away, +and the context will be lost if not recorded in the commit message that +gets squashed into. + +Run affected tests before finalizing. + +### `amend!` Commits + +A `fixup!` commit keeps the target's commit message and merely combines +its diff into the target. An `amend!` commit additionally **replaces** +the target's commit message with its own body. Use `amend!` when the +fix changes the meaning of the target sufficiently that the original +subject or body is no longer accurate, or when the goal is to align a +downstream commit with a specific upstream replacement. + +The format is rigid: the first line of an `amend!` commit must be +exactly `amend! `, followed by a blank line and then +the **new** commit message that should replace the target's, starting +with the new subject line: + +``` +amend! mingw: use mimalloc + +mingw: stop using nedmalloc + +The vendored nedmalloc allocator under compat/nedmalloc/ has been +unmaintained upstream... +``` + +After autosquash, the resulting commit has the new subject (`mingw: +stop using nedmalloc`), the new body, and a diff that is the +composition of the target's diff and the `amend!`'s diff. Crafting the +`amend!` diff so that the composition equals a known upstream commit's +diff is the canonical way to align a downstream branch-thicket commit +with an in-flight upstream replacement: when the next merging-rebase +picks up the upstream commit, the byte-identical downstream commit +collapses into it cleanly. + +### PRs Composed Entirely of `fixup!` and `amend!` Commits + +Adjusting or removing a feature that lives in the branch thicket is +often best expressed as a PR that consists *only* of `fixup!` and +`amend!` commits targeting the existing thicket commits. Each pair +autosquashes during the next merging-rebase. Pairs whose diffs cancel +exactly produce empty commits, which the rebase drops with +`--empty=drop`. The end state is *as if the original commits had been +edited or removed in place*, while preserving review-friendly atomic +patches in the PR. + +This is the preferred pattern for reverting a multi-commit downstream +feature. Order the fixups in **reverse** of the originals so each +revert applies cleanly to the worktree as you build the series. + +### Common Adaptation Patterns + +**Struct field moves**: When upstream moves fields between structs, update +all downstream code that accesses those fields. + +**API changes**: When upstream changes function signatures, update callers +and verify semantics are preserved. + +**New abstractions**: When upstream introduces new layers, ensure downstream +code uses the correct instance. + +## Coding Conventions + +The Git project maintains a charmingly old-school, Unix-greybeard aesthetic +when it comes to text encoding. In the spirit of the PDP-11 and Bell Labs +terminal sessions of yore: + +- **ASCII only**: Avoid Unicode characters in source code, comments, and + documentation. Use `->` instead of `→`, `--` instead of `—`, and so on. + To verify your changes contain no non-ASCII characters: + ``` + git diff | LC_ALL=C grep '[^ -~]' + ``` +- **80 columns per line**: The mailing list veterans will "kindly" remind you + that lines should not exceed 80 characters (they do mean columns, but + let's not split beards or hairs about wide glyphs). + First, check for whitespace errors (trailing whitespace, mid-line tabs, etc.): + ``` + git diff --check + ``` + Once that passes, you know tabs only appear at line beginnings, so each + tab equals exactly 8 columns. To find lines exceeding 80 columns: + ``` + git diff --no-color | grep '^+' | sed 's/\t/ /g' | grep '.\{82\}' + ``` + (We use 82 because diff output prefixes added lines with `+`.) +- **Tabs for indentation**: The codebase uses tabs, not spaces. +- **No trailing whitespace**: Clean up your lines. + +**Pre-commit checklist.** Run all three checks before every commit: + +```bash +git diff --check && +git diff --no-color | LC_ALL=C grep '[^ -~]' && + echo "ERROR: non-ASCII characters found" && +git diff --no-color | grep '^+' | sed 's/\t/ /g' | + grep '.\{82\}' && + echo "ERROR: lines exceed 80 columns" +``` + +The first command catches whitespace errors. If either of the latter +two produces output, fix the offending lines before committing. Note +that these checks apply to commit messages as well (wrap at 76 columns +for messages, 80 for code). + +See `Documentation/CodingGuidelines` for the full set of conventions. + +### strbuf patterns + +Use `strbuf_addf()` with string continuation for multi-line content instead +of multiple `strbuf_addstr()` calls: + +```c +/* Good */ +strbuf_addf(&buf, + "tree %s\n" + "author %s\n" + "committer %s\n" + "\ncommit message\n", + tree_hex, author, committer); + +/* Avoid */ +strbuf_addstr(&buf, "tree "); +strbuf_addstr(&buf, tree_hex); +strbuf_addstr(&buf, "\nauthor "); +/* ... */ +``` + +Choose descriptive variable names (`header` for pack headers, not generic +`buf`; use `buf` for the secondary strbuf if you cannot reuse the first). + +## Platform Considerations + +### Windows-specific issues + +On Windows, `unsigned long` is 32 bits even on 64-bit systems. Use `size_t` +for sizes that may exceed 4GB. Be careful with format strings: use `PRIuMAX` +with a cast for `size_t` values. + +## Contributing to Git for Windows + +The primary contribution path for this fork is a PR against +`git-for-windows/git`'s `main` branch. The repository is laid out as a +branch thicket on top of an upstream Git base; see +[Merging-Rebases](#merging-rebases) and +[Analyzing Branch Thickets](#analyzing-branch-thickets) for the +mechanics. + +### Opening a PR + +Push the topic branch to a personal fork on GitHub, then: + +```bash +gh pr create \ + --repo git-for-windows/git \ + --base main \ + --head : \ + --title "" \ + --body-file +``` + +Unlike upstream contributions, the PR body is rendered as Markdown on +GitHub, not sent as email. Use the formatting that aids review: +fenced code blocks, tables, links to workflow runs. + +### When the PR Adjusts the Thicket Itself + +If the PR's purpose is to edit, remove, or replace existing +branch-thicket commits, the natural form is a series of `fixup!` or +`amend!` commits targeting the affected originals. See +[Fixup Commits](#fixup-commits), +[`amend!` Commits](#amend-commits), and +[PRs Composed Entirely of `fixup!` and `amend!` Commits](#prs-composed-entirely-of-fixup-and-amend-commits). +The merging-rebase that produces the next `main` autosquashes these +into the thicket; the PR exists for review of the individual +adjustments. + +### When an Upstream Patch Will Replace a Thicket Commit + +If an upstream patch is in flight (for instance, on `gitgitgadget/git` +in `seen` or `next`) that replaces a downstream thicket commit, an +`amend!` commit whose body is a verbatim copy of the upstream commit +message and whose diff aligns the autosquashed target with the +upstream commit's diff is the canonical pattern. The next +merging-rebase that picks up the upstream commit will recognize the +two as byte-identical and collapse them. + +## Contributing to Upstream Git via GitGitGadget + +### Overview + +The upstream Git project accepts contributions via the mailing list +(`git@vger.kernel.org`). [GitGitGadget](https://gitgitgadget.github.io/) +bridges GitHub PRs to the mailing list: you push a branch to your GitHub +fork, open a PR against https://github.com/gitgitgadget/git, and +GitGitGadget formats and sends the patches. + +### Workflow + +1. Push the topic branch to your personal fork on GitHub (the remote + that points at `https://github.com//git`). +2. Open a PR from `:` against `gitgitgadget/git`'s `master`. +3. The PR title becomes the patch series subject; the PR body becomes the + cover letter. Use + `gh pr create --repo gitgitgadget/git --head :`. +4. Use `/submit` as a PR comment to send patches to the mailing list. +5. After review feedback, update the branch, force-push, and `/submit` again. + +### Branch Naming + +Do **not** use an initials prefix (like `ds/` or `js/`). That convention is +used by the Git maintainer when picking up topics, not by contributors. Use +descriptive names like `tests-explicit-bare-repo`. + +### Cover Letter Style + +The PR body is the cover letter. It should be plain text (not Markdown with +headers or bullet formatting), since it will be sent as email. Structure: + +- A brief subject line (the PR title, e.g. "tests: access bare repositories + explicitly") +- Motivation: why is this change needed? +- Summary: what does the series do? What patterns/techniques does it use? +- Scope: is this part of a larger effort? If so, link to the tracking PR. + +Keep it factual and measured. Avoid framing changes in terms of security +when contributing to upstream Git; frame them as robustness, correctness, +or preparation for future defaults. + +### Commit Message Conventions (Upstream Git) + +Upstream Git commit messages follow stricter conventions than the Microsoft +Git fork: + +- **Subject line**: `: ` (lowercase after the colon). + The `` is typically a file name without extension (e.g. `t0001`, + `setup`, `scalar`) or a subsystem name (e.g. `tests`, `refs`). +- **Body**: Flowing English prose, no bullet points. Wrap at 76 columns. +- **ASCII only**: No Unicode characters anywhere in the message. +- **Trailers**: `Signed-off-by` is mandatory. `Assisted-by` for AI. +- The subject line must accurately describe the diff content. If a commit + adds `--git-dir=.` to one invocation, do not title it "wrap bare repo + commands in subshell with `GIT_DIR`". + +### Patch Series with Dependencies + +When contributing a branch thicket (multiple related patch series with +dependencies), submit the foundation series first and note the overall +effort in the cover letter with a link to the tracking PR or `compare` +URL. Submit dependent series after earlier ones land in `seen`. + +Use `git replay --onto ..` to test whether a +sub-branch applies cleanly to a given base (e.g., `upstream/master` or +`upstream/seen`) without touching the working tree. By default (since +the `--ref-action` default changed to `update`), `git replay` updates +named refs in the range directly, producing no stdout output. Use +`--ref-action=print` to get the old behavior of printing `update-ref` +commands to stdout instead. Always verify that `git replay` actually +did something by checking the reflog of the affected branches. + +## Working with Worktrees + +### General Principles + +Use worktrees to work on multiple topics simultaneously without stashing +or switching branches. Keep worktrees as subdirectories of the main +repository and add them to `.git/info/exclude` so they do not show up +as untracked files. + +```bash +git worktree add +echo "" >> .git/info/exclude +``` + +### Rewriting Commits with `--update-refs` + +When rewriting history in a worktree (e.g., fixing a commit message via +`amend!` + autosquash), use `--update-refs` so that other local branches +pointing into the rewritten range are updated automatically: + +```bash +# Create a local branch at the commit to be pushed +git branch + +# Create the amend! commit and autosquash +git commit --allow-empty -F +GIT_SEQUENCE_EDITOR=true GIT_EDITOR=true \ + git rebase -i --autosquash --update-refs + +# Verify: tree should be identical +git diff @{1}.. + +# Force-push the updated branch +git push --force-with-lease +``` + +The `--update-refs` flag is essential: without it, only the checked-out +branch is rewritten and other branches become stale, pointing at +pre-rewrite commits. + +### Verifying Rebase Results + +After any rebase, verify that the tree content is unchanged (unless you +intentionally modified it): + +```bash +git diff @{1} # Should be empty for pure rewording +git range-diff @{1}... # Shows per-commit changes +``` + +## Analyzing Branch Thickets + +When a branch is structured as a sequence of merged sub-branches (a +"branch thicket"), use the merge structure to extract sub-branches: + +```bash +# List the merge commits (sub-branches) +git log --oneline --first-parent ...upstream/master | grep 'Merge branch' + +# Extract commits for a specific sub-branch (second parent of its merge) +git log --oneline ^1..^2 + +# Find what each sub-branch forks from +git log -1 --format='%H %s' ^ +``` + +Use `git replay` to test whether sub-branches can be rebased onto a new +base without conflicts. This replaces speculation about "overlapping files" +with actual evidence: + +```bash +git replay --onto upstream/master .. +``` + +If the range contains merge commits, `git replay` will fail with "replaying +merge commits is not supported yet!" In that case, identify the linear +commit range and replay just those commits. + +## Resources + +- [Git for Windows](https://gitforwindows.org/) +- [Git Internals](https://git-scm.com/book/en/v2/Git-Internals-Plumbing-and-Porcelain) +- [GitGitGadget](https://gitgitgadget.github.io/) - Bridge GitHub PRs to + the Git mailing list +- [Git Mailing List Archive](https://lore.kernel.org/git/) - Searchable + archive of all upstream discussion From a5ca7eab6e695c9c058a3d8b38bf0c25d14be033 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 11 Oct 2019 13:22:24 +0200 Subject: [PATCH 220/229] Modify the Code of Conduct for Git for Windows The Git project followed Git for Windows' lead and added their Code of Conduct, based on the Contributor Covenant v1.4, later updated to v2.0. We adapt it slightly to Git for Windows. Signed-off-by: Johannes Schindelin --- CODE_OF_CONDUCT.md | 58 +++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index e58917c50a96dc..4daef7e3ce9196 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,9 +1,9 @@ -# Git Code of Conduct +# Git for Windows Code of Conduct This code of conduct outlines our expectations for participants within -the Git community, as well as steps for reporting unacceptable behavior. -We are committed to providing a welcoming and inspiring community for -all and expect our code of conduct to be honored. Anyone who violates +the **Git for Windows** community, as well as steps for reporting unacceptable +behavior. We are committed to providing a welcoming and inspiring community +for all and expect our code of conduct to be honored. Anyone who violates this code of conduct may be banned from the community. ## Our Pledge @@ -12,8 +12,8 @@ We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, -nationality, personal appearance, race, religion, or sexual identity -and orientation. +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. @@ -28,17 +28,17 @@ community include: * Giving and gracefully accepting constructive feedback * Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience -* Focusing on what is best not just for us as individuals, but for the - overall community +* Focusing on what is best not just for us as individuals, but for the overall + community Examples of unacceptable behavior include: -* The use of sexualized language or imagery, and sexual attention or - advances of any kind +* The use of sexualized language or imagery, and sexual attention or advances of + any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission +* Publishing others' private information, such as a physical or email address, + without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting @@ -58,20 +58,14 @@ decisions when appropriate. This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. -Examples of representing our community include using an official e-mail address, +Examples of representing our community include using an official email address, posting via an official social media account, or acting as an appointed representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement at -git@sfconservancy.org, or individually: - - - Ævar Arnfjörð Bjarmason - - Christian Couder - - Junio C Hamano - - Taylor Blau +reported by contacting the Git for Windows maintainer. All complaints will be reviewed and investigated promptly and fairly. @@ -94,15 +88,15 @@ behavior was inappropriate. A public apology may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series -of actions. +**Community Impact**: A violation through a single incident or series of +actions. **Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels -like social media. Violating these terms may lead to a temporary or -permanent ban. +like social media. Violating these terms may lead to a temporary or permanent +ban. ### 3. Temporary Ban @@ -118,27 +112,27 @@ Violating these terms may lead to a permanent ban. ### 4. Permanent Ban **Community Impact**: Demonstrating a pattern of violation of community -standards, including sustained inappropriate behavior, harassment of an +standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. -**Consequence**: A permanent ban from any sort of public interaction within -the community. +**Consequence**: A permanent ban from any sort of public interaction within the +community. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], -version 2.0, available at -[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0]. +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. For answers to common questions about this code of conduct, see the FAQ at -[https://www.contributor-covenant.org/faq][FAQ]. Translations are available -at [https://www.contributor-covenant.org/translations][translations]. +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. [homepage]: https://www.contributor-covenant.org -[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html [Mozilla CoC]: https://github.com/mozilla/diversity [FAQ]: https://www.contributor-covenant.org/faq [translations]: https://www.contributor-covenant.org/translations From 7d7c08cb62ebb56756eb217a2e58b5ccfb5d9025 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 1 Mar 2018 12:10:14 -0500 Subject: [PATCH 221/229] CONTRIBUTING.md: add guide for first-time contributors Getting started contributing to Git can be difficult on a Windows machine. CONTRIBUTING.md contains a guide to getting started, including detailed steps for setting up build tools, running tests, and submitting patches to upstream. [includes an example by Pratik Karki how to submit v2, v3, v4, etc.] Signed-off-by: Derrick Stolee --- CONTRIBUTING.md | 417 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 417 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000000000..fbe729e43baf86 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,417 @@ +How to Contribute to Git for Windows +==================================== + +Git was originally designed for Unix systems and still today, all the build tools for the Git +codebase assume you have standard Unix tools available in your path. If you have an open-source +mindset and want to start contributing to Git, but primarily use a Windows machine, then you may +have trouble getting started. This guide is for you. + +Get the Source +-------------- + +Clone the [GitForWindows repository on GitHub](https://github.com/git-for-windows/git). +It is helpful to create your own fork for storing your development branches. + +Windows uses different line endings than Unix systems. See +[this GitHub article on working with line endings](https://help.github.com/articles/dealing-with-line-endings/#refreshing-a-repository-after-changing-line-endings) +if you have trouble with line endings. + +Build the Source +---------------- + +First, download and install the latest [Git for Windows SDK (64-bit)](https://github.com/git-for-windows/build-extra/releases/latest). +When complete, you can run the Git SDK, which creates a new Git Bash terminal window with +the additional development commands, such as `make`. + + As of time of writing, the SDK uses a different credential manager, so you may still want to use normal Git + Bash for interacting with your remotes. Alternatively, use SSH rather than HTTPS and + avoid credential manager problems. + +You should now be ready to type `make` from the root of your `git` source directory. +Here are some helpful variations: + +* `make -j[N] DEVELOPER=1`: Compile new sources using up to N concurrent processes. + The `DEVELOPER` flag turns on all warnings; code failing these warnings will not be + accepted upstream ("upstream" = "the core Git project"). +* `make clean`: Delete all compiled files. + +When running `make`, you can use `-j$(nproc)` to automatically use the number of processors +on your machine as the number of concurrent build processes. + +You can go deeper on the Windows-specific build process by reading the +[technical overview](https://gitforwindows.org/technical-overview) or the +[guide to compiling Git with Visual Studio](https://gitforwindows.org/compiling-git-with-visual-studio). + +## Building `git` on Windows with Visual Studio + +The typical approach to building `git` is to use the standard `Makefile` with GCC, as +above. Developers working in a Windows environment may want to instead build with the +[Microsoft Visual C++ compiler and libraries toolset (MSVC)](https://blogs.msdn.microsoft.com/vcblog/2017/03/07/msvc-the-best-choice-for-windows/). +There are a few benefits to using MSVC over GCC during your development, including creating +symbols for debugging and [performance tracing](https://github.com/Microsoft/perfview#perfview-overview). + +There are two ways to build Git for Windows using MSVC. Each have their own merits. + +### Using SDK Command Line + +Use one of the following commands from the SDK Bash window to build Git for Windows: + +``` + make MSVC=1 -j12 + make MSVC=1 DEBUG=1 -j12 +``` + +The first form produces release-mode binaries; the second produces debug-mode binaries. +Both forms produce PDB files and can be debugged. However, the first is best for perf +tracing and the second is best for single-stepping. + +You can then open Visual Studio and select File -> Open -> Project/Solution and select +the compiled `git.exe` file. This creates a basic solution and you can use the debugging +and performance tracing tools in Visual Studio to monitor a Git process. Use the Debug +Properties page to set the working directory and command line arguments. + +Be sure to clean up before switching back to GCC (or to switch between debug and +release MSVC builds): + +``` + make MSVC=1 -j12 clean + make MSVC=1 DEBUG=1 -j12 clean +``` + +### Using the IDE + +If you prefer working in Visual Studio with a solution full of projects, then you can use +CMake, either by letting Visual Studio configure it automatically (simply open Git's +top-level directory via `File>Open>Folder...`) or by (downloading and) running +[CMake](https://cmake.org) manually. + +What to Change? +--------------- + +Many new contributors ask: What should I start working on? + +One way to win big with the open-source community is to look at the +[issues page](https://github.com/git-for-windows/git/issues) and see if there are any issues that +you can fix quickly, or if anything catches your eye. + +You can also look at [the unofficial Chromium issues page](https://crbug.com/git) for +multi-platform issues. You can look at recent user questions on +[the Git mailing list](https://public-inbox.org/git). + +Or you can "scratch your own itch", i.e. address an issue you have with Git. The team at Microsoft where the Git for Windows maintainer works, for example, is focused almost entirely on [improving performance](https://blogs.msdn.microsoft.com/devops/2018/01/11/microsofts-performance-contributions-to-git-in-2017/). +We approach our work by finding something that is slow and try to speed it up. We start our +investigation by reliably reproducing the slow behavior, then running that example using +the MSVC build and tracing the results in PerfView. + +You could also think of something you wish Git could do, and make it do that thing! The +only concern I would have with this approach is whether or not that feature is something +the community also wants. If this excites you though, go for it! Don't be afraid to +[get involved in the mailing list](http://vger.kernel.org/vger-lists.html#git) early for +feedback on the idea. + +Test Your Changes +----------------- + +After you make your changes, it is important that you test your changes. Manual testing is +important, but checking and extending the existing test suite is even more important. You +want to run the functional tests to see if you broke something else during your change, and +you want to extend the functional tests to be sure no one breaks your feature in the future. + +### Functional Tests + +Navigate to the `t/` directory and type `make` to run all tests or use `prove` as +[described on this Git for Windows page](https://gitforwindows.org/building-git): + +``` +prove -j12 --state=failed,save ./t[0-9]*.sh +``` + +You can also run each test directly by running the corresponding shell script with a name +like `tNNNN-descriptor.sh`. + +If you are adding new functionality, you may need to create unit tests by creating +helper commands that test a very limited action. These commands are stored in `t/helpers`. +When adding a helper, be sure to add a line to `t/Makefile` and to the `.gitignore` for the +binary file you add. The Git community prefers functional tests using the full `git` +executable, so try to exercise your new code using `git` commands before creating a test +helper. + +To find out why a test failed, repeat the test with the `-x -v -d -i` options and then +navigate to the appropriate "trash" directory to see the data shape that was used for the +test failed step. + +Read [`t/README`](t/README) for more details. + +### Performance Tests + +If you are working on improving performance, you will need to be acquainted with the +performance tests in `t/perf`. There are not too many performance tests yet, but adding one +as your first commit in a patch series helps to communicate the boost your change provides. + +To check the change in performance across multiple versions of `git`, you can use the +`t/perf/run` script. For example, to compare the performance of `git rev-list` across the +`core/master` and `core/next` branches compared to a `topic` branch, you can run + +``` +cd t/perf +./run core/master core/next topic -- p0001-rev-list.sh +``` + +You can also set certain environment variables to help test the performance on different +repositories or with more repetitions. The full list is available in +[the `t/perf/README` file](t/perf/README), +but here are a few important ones: + +``` +GIT_PERF_REPO=/path/to/repo +GIT_PERF_LARGE_REPO=/path/to/large/repo +GIT_PERF_REPEAT_COUNT=10 +``` + +When running the performance tests on Linux, you may see a message "Can't locate JSON.pm in +@INC" and that means you need to run `sudo cpanm install JSON` to get the JSON perl package. + +For running performance tests, it can be helpful to set up a few repositories with strange +data shapes, such as: + +**Many objects:** Clone repos such as [Kotlin](https://github.com/jetbrains/kotlin), [Linux](https://github.com/torvalds/linux), or [Android](https://source.android.com/setup/downloading). + +**Many pack-files:** You can split a fresh clone into multiple pack-files of size at most +16MB by running `git repack -adfF --max-pack-size=16m`. See the +[`git repack` documentation](https://git-scm.com/docs/git-repack) for more information. +You can count the number of pack-files using `ls .git/objects/pack/*.pack | wc -l`. + +**Many loose objects:** If you already split your repository into multiple pack-files, then +you can pick one to split into loose objects using `cat .git/objects/pack/[id].pack | git unpack-objects`; +delete the `[id].pack` and `[id].idx` files after this. You can count the number of loose +bjects using `ls .git/objects/??/* | wc -l`. + +**Deep history:** Usually large repositories also have deep histories, but you can use the +[test-many-commits-1m repo](https://github.com/cirosantilli/test-many-commits-1m/) to +target deep histories without the overhead of many objects. One issue with this repository: +there are no merge commits, so you will need to use a different repository to test a "wide" +commit history. + +**Large Index:** You can generate a large index and repo by using the scripts in +`t/perf/repos`. There are two scripts. `many-files.sh` which will generate a repo with +same tree and blobs but different paths. Using `many-files.sh -d 5 -w 10 -f 9` will create +a repo with ~1 million entries in the index. `inflate-repo.sh` will use an existing repo +and copy the current work tree until it is a specified size. + +Test Your Changes on Linux +-------------------------- + +It can be important to work directly on the [core Git codebase](https://github.com/git/git), +such as a recent commit into the `master` or `next` branch that has not been incorporated +into Git for Windows. Also, it can help to run functional and performance tests on your +code in Linux before submitting patches to the mailing list, which focuses on many platforms. +The differences between Windows and Linux are usually enough to catch most cross-platform +issues. + +### Using the Windows Subsystem for Linux + +The [Windows Subsystem for Linux (WSL)](https://docs.microsoft.com/en-us/windows/wsl/install-win10) +allows you to [install Ubuntu Linux as an app](https://www.microsoft.com/en-us/store/p/ubuntu/9nblggh4msv6) +that can run Linux executables on top of the Windows kernel. Internally, +Linux syscalls are interpreted by the WSL, everything else is plain Ubuntu. + +First, open WSL (either type "Bash" in Cortana, or execute "bash.exe" in a CMD window). +Then install the prerequisites, and `git` for the initial clone: + +``` +sudo apt-get update +sudo apt-get install git gcc make libssl-dev libcurl4-openssl-dev \ + libexpat-dev tcl tk gettext git-email zlib1g-dev +``` + +Then, clone and build: + +``` +git clone https://github.com/git-for-windows/git +cd git +git remote add -f upstream https://github.com/git/git +make +``` + +Be sure to clone into `/home/[user]/` and not into any folder under `/mnt/?/` or your build +will fail due to colons in file names. + +### Using a Linux Virtual Machine with Hyper-V + +If you prefer, you can use a virtual machine (VM) to run Linux and test your changes in the +full environment. The test suite runs a lot faster on Linux than on Windows or with the WSL. +You can connect to the VM using an SSH terminal like +[PuTTY](https://www.chiark.greenend.org.uk/~sgtatham/putty/). + +The following instructions are for using Hyper-V, which is available in some versions of Windows. +There are many virtual machine alternatives available, if you do not have such a version installed. + +* [Download an Ubuntu Server ISO](https://www.ubuntu.com/download/server). +* Open [Hyper-V Manager](https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/quick-start/enable-hyper-v). +* [Set up a virtual switch](https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/quick-start/connect-to-network) + so your VM can reach the network. +* Select "Quick Create", name your machine, select the ISO as installation source, and un-check + "This virtual machine will run Windows." +* Go through the Ubuntu install process, being sure to select to install OpenSSH Server. +* When install is complete, log in and check the SSH server status with `sudo service ssh status`. + * If the service is not found, install with `sudo apt-get install openssh-server`. + * If the service is not running, then use `sudo service ssh start`. +* Use `shutdown -h now` to shutdown the VM, go to the Hyper-V settings for the VM, expand Network Adapter + to select "Advanced Features", and set the MAC address to be static (this can save your VM from losing + network if shut down incorrectly). +* Provide as many cores to your VM as you can (for parallel builds). +* Restart your VM, but do not connect. +* Use `ssh` in Git Bash, download [PuTTY](https://www.chiark.greenend.org.uk/~sgtatham/putty/), or use your favorite SSH client to connect to the VM through SSH. + +In order to build and use `git`, you will need the following libraries via `apt-get`: + +``` +sudo apt-get update +sudo apt-get install git gcc make libssl-dev libcurl4-openssl-dev \ + libexpat-dev tcl tk gettext git-email zlib1g-dev +``` + +To get your code from your Windows machine to the Linux VM, it is easiest to push the branch to your fork of Git and clone your fork in the Linux VM. + +Don't forget to set your `git` config with your preferred name, email, and editor. + +Polish Your Commits +------------------- + +Before submitting your patch, be sure to read the [coding guidelines](https://github.com/git/git/blob/master/Documentation/CodingGuidelines) +and check your code to match as best you can. This can be a lot of effort, but it saves +time during review to avoid style issues. + +The other possibly major difference between the mailing list submissions and GitHub PR workflows +is that each commit will be reviewed independently. Even if you are submitting a +patch series with multiple commits, each commit must stand on it's own and be reviewable +by itself. Make sure the commit message clearly explain the why of the commit not the how. +Describe what is wrong with the current code and how your changes have made the code better. + +When preparing your patch, it is important to put yourself in the shoes of the Git community. +Accepting a patch requires more justification than approving a pull request from someone on +your team. The community has a stable product and is responsible for keeping it stable. If +you introduce a bug, then they cannot count on you being around to fix it. When you decided +to start work on a new feature, they were not part of the design discussion and may not +even believe the feature is worth introducing. + +Questions to answer in your patch message (and commit messages) may include: +* Why is this patch necessary? +* How does the current behavior cause pain for users? +* What kinds of repositories are necessary for noticing a difference? +* What design options did you consider before writing this version? Do you have links to + code for those alternate designs? +* Is this a performance fix? Provide clear performance numbers for various well-known repos. + +Here are some other tips that we use when cleaning up our commits: + +* Commit messages should be wrapped at 76 columns per line (or less; 72 is also a + common choice). +* Make sure the commits are signed off using `git commit (-s|--signoff)`. See + [SubmittingPatches](https://github.com/git/git/blob/v2.8.1/Documentation/SubmittingPatches#L234-L286) + for more details about what this sign-off means. +* Check for whitespace errors using `git diff --check [base]...HEAD` or `git log --check`. +* Run `git rebase --whitespace=fix` to correct upstream issues with whitespace. +* Become familiar with interactive rebase (`git rebase -i`) because you will be reordering, + squashing, and editing commits as your patch or series of patches is reviewed. +* Make sure any shell scripts that you add have the executable bit set on them. This is + usually for test files that you add in the `/t` directory. You can use + `git add --chmod=+x [file]` to update it. You can test whether a file is marked as executable + using `git ls-files --stage \*.sh`; the first number is 100755 for executable files. +* Your commit titles should match the "area: change description" format. Rules of thumb: + * Choose ": " prefix appropriately. + * Keep the description short and to the point. + * The word that follows the ": " prefix is not capitalized. + * Do not include a full-stop at the end of the title. + * Read a few commit messages -- using `git log origin/master`, for instance -- to + become acquainted with the preferred commit message style. +* Build source using `make DEVELOPER=1` for extra-strict compiler warnings. + +Submit Your Patch +----------------- + +Git for Windows [accepts pull requests on GitHub](https://github.com/git-for-windows/git/pulls), but +these are reserved for Windows-specific improvements. For core Git, submissions are accepted on +[the Git mailing list](https://public-inbox.org/git). + +### Configure Git to Send Emails + +There are a bunch of options for configuring the `git send-email` command. These options can +be found in the documentation for +[`git config`](https://git-scm.com/docs/git-config) and +[`git send-email`](https://git-scm.com/docs/git-send-email). + +``` +git config --global sendemail.smtpserver +git config --global sendemail.smtpserverport 587 +git config --global sendemail.smtpencryption tls +git config --global sendemail.smtpuser +``` + +To avoid storing your password in the config file, store it in the Git credential manager: + +``` +$ git credential fill +protocol=smtp +host= +username= +password=password +``` + +Before submitting a patch, read the [Git documentation on submitting patches](https://github.com/git/git/blob/master/Documentation/SubmittingPatches). + +To construct a patch set, use the `git format-patch` command. There are three important options: + +* `--cover-letter`: If specified, create a `[v#-]0000-cover-letter.patch` file that can be + edited to describe the patch as a whole. If you previously added a branch description using + `git branch --edit-description`, you will end up with a 0/N mail with that description and + a nice overall diffstat. +* `--in-reply-to=[Message-ID]`: This will mark your cover letter as replying to the given + message (which should correspond to your first iteration). To determine the correct Message-ID, + find the message you are replying to on [public-inbox.org/git](https://public-inbox.org/git) and take + the ID from between the angle brackets. + +* `--subject-prefix=[prefix]`: This defaults to [PATCH]. For subsequent iterations, you will want to + override it like `--subject-prefix="[PATCH v2]"`. You can also use the `-v` option to have it + automatically generate the version number in the patches. + +If you have multiple commits and use the `--cover-letter` option be sure to open the +`0000-cover-letter.patch` file to update the subject and add some details about the overall purpose +of the patch series. + +### Examples + +To generate a single commit patch file: +``` +git format-patch -s -o [dir] -1 +``` +To generate four patch files from the last three commits with a cover letter: +``` +git format-patch --cover-letter -s -o [dir] HEAD~4 +``` +To generate version 3 with four patch files from the last four commits with a cover letter: +``` +git format-patch --cover-letter -s -o [dir] -v 3 HEAD~4 +``` + +### Submit the Patch + +Run [`git send-email`](https://git-scm.com/docs/git-send-email), starting with a test email: + +``` +git send-email --to=yourself@address.com [dir with patches]/*.patch +``` + +After checking the receipt of your test email, you can send to the list and to any +potentially interested reviewers. + +``` +git send-email --to=git@vger.kernel.org --cc= --cc= [dir with patches]/*.patch +``` + +To submit a nth version patch (say version 3): + +``` +git send-email --to=git@vger.kernel.org --cc= --cc= \ + --in-reply-to= [dir with patches]/*.patch +``` From bbc1a48393ab443d54b8d1418b866ad7a16e465c Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 10 Jan 2014 16:16:03 -0600 Subject: [PATCH 222/229] README.md: Add a Windows-specific preamble MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Includes touch-ups by 마누엘, Philip Oakley and 孙卓识. Signed-off-by: Johannes Schindelin --- README.md | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 46489b0971d04d..026d5d85caef09 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,77 @@ -[![GitHub build status](https://github.com/git/git/workflows/CI/badge.svg)](https://github.com/git/git/actions?query=branch%3Amaster+event%3Apush) -[![GitLab build status](https://gitlab.com/git-scm/git/badges/master/pipeline.svg)](https://gitlab.com/git-scm/git/-/pipelines?ref=master) +Git for Windows +=============== + +[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) +[![Open in Visual Studio Code](https://img.shields.io/static/v1?logo=visualstudiocode&label=&message=Open%20in%20Visual%20Studio%20Code&labelColor=2c2c32&color=007acc&logoColor=007acc)](https://open.vscode.dev/git-for-windows/git) +[![Build status](https://github.com/git-for-windows/git/workflows/CI/badge.svg)](https://github.com/git-for-windows/git/actions?query=branch%3Amain+event%3Apush) +[![Join the chat at https://gitter.im/git-for-windows/git](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/git-for-windows/git?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + +This is [Git for Windows](http://git-for-windows.github.io/), the Windows port +of [Git](http://git-scm.com/). + +The Git for Windows project is run using a [governance +model](http://git-for-windows.github.io/governance-model.html). If you +encounter problems, you can report them as [GitHub +issues](https://github.com/git-for-windows/git/issues), discuss them in Git +for Windows' [Discussions](https://github.com/git-for-windows/git/discussions) +or on the [Git mailing list](mailto:git@vger.kernel.org), and [contribute bug +fixes](https://gitforwindows.org/how-to-participate). + +To build Git for Windows, please either install [Git for Windows' +SDK](https://gitforwindows.org/#download-sdk), start its `git-bash.exe`, `cd` +to your Git worktree and run `make`, or open the Git worktree as a folder in +Visual Studio. + +To verify that your build works, use one of the following methods: + +- If you want to test the built executables within Git for Windows' SDK, + prepend `/bin-wrappers` to the `PATH`. +- Alternatively, run `make install` in the Git worktree. +- If you need to test this in a full installer, run `sdk build + git-and-installer`. +- You can also "install" Git into an existing portable Git via `make install + DESTDIR=` where `` refers to the top-level directory of the + portable Git. In this instance, you will want to prepend that portable Git's + `/cmd` directory to the `PATH`, or test by running that portable Git's + `git-bash.exe` or `git-cmd.exe`. +- If you built using a recent Visual Studio, you can use the menu item + `Build>Install git` (you will want to click on `Project>CMake Settings for + Git` first, then click on `Edit JSON` and then point `installRoot` to the + `mingw64` directory of an already-unpacked portable Git). + + As in the previous bullet point, you will then prepend `/cmd` to the `PATH` + or run using the portable Git's `git-bash.exe` or `git-cmd.exe`. +- If you want to run the built executables in-place, but in a CMD instead of + inside a Bash, you can run a snippet like this in the `git-bash.exe` window + where Git was built (ensure that the `EOF` line has no leading spaces), and + then paste into the CMD window what was put in the clipboard: + + ```sh + clip.exe < (see https://subspace.kernel.org/subscribing.html for details). The mailing list archives are available at , and other archival sites. +The core git mailing list is plain text (no HTML!). Issues which are security relevant should be disclosed privately to the Git Security mailing list . From 6e4c4ac8178ff4dd41b4ded4972ffb8c27e07f95 Mon Sep 17 00:00:00 2001 From: Brendan Forster Date: Thu, 18 Feb 2016 21:29:50 +1100 Subject: [PATCH 223/229] Add an issue template With improvements by Clive Chan, Adric Norris, Ben Bodenmiller and Philip Oakley. Helped-by: Clive Chan Helped-by: Adric Norris Helped-by: Ben Bodenmiller Helped-by: Philip Oakley Signed-off-by: Brendan Forster Signed-off-by: Johannes Schindelin --- .github/ISSUE_TEMPLATE/bug-report.yml | 105 ++++++++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 1 + 2 files changed, 106 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug-report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml diff --git a/.github/ISSUE_TEMPLATE/bug-report.yml b/.github/ISSUE_TEMPLATE/bug-report.yml new file mode 100644 index 00000000000000..b49593339932b2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yml @@ -0,0 +1,105 @@ +name: Bug report +description: Use this template to report bugs. +body: + - type: checkboxes + id: search + attributes: + label: Existing issues matching what you're seeing + description: Please search for [open](https://github.com/git-for-windows/git/issues?q=is%3Aopen) or [closed](https://github.com/git-for-windows/git/issues?q=is%3Aclosed) issue matching what you're seeing before submitting a new issue. + options: + - label: I was not able to find an open or closed issue matching what I'm seeing + - type: textarea + id: git-for-windows-version + attributes: + label: Git for Windows version + description: Which version of Git for Windows are you using? + placeholder: Please insert the output of `git --version --build-options` here + render: shell + validations: + required: true + - type: dropdown + id: windows-version + attributes: + label: Windows version + description: Which version of Windows are you running? + options: + - Windows 8.1 + - Windows 10 + - Windows 11 + - Other + default: 2 + validations: + required: true + - type: dropdown + id: windows-arch + attributes: + label: Windows CPU architecture + description: What CPU Archtitecture does your Windows target? + options: + - i686 (32-bit) + - x86_64 (64-bit) + - ARM64 + default: 1 + validations: + required: true + - type: textarea + id: windows-version-cmd + attributes: + label: Additional Windows version information + description: This provides us with further information about your Windows such as the build number + placeholder: Please insert the output of `cmd.exe /c ver` here + render: shell + - type: textarea + id: options + attributes: + label: Options set during installation + description: What options did you set as part of the installation? Or did you choose the defaults? + placeholder: | + One of the following: + > type "C:\Program Files\Git\etc\install-options.txt" + > type "C:\Program Files (x86)\Git\etc\install-options.txt" + > type "%USERPROFILE%\AppData\Local\Programs\Git\etc\install-options.txt" + > type "$env:USERPROFILE\AppData\Local\Programs\Git\etc\install-options.txt" + $ cat /etc/install-options.txt + render: shell + validations: + required: true + - type: textarea + id: other-things + attributes: + label: Other interesting things + description: Any other interesting things about your environment that might be related to the issue you're seeing? + - type: input + id: terminal + attributes: + label: Terminal/shell + description: Which terminal/shell are you running Git from? e.g Bash/CMD/PowerShell/other + validations: + required: true + - type: textarea + id: commands + attributes: + label: Commands that trigger the issue + description: What commands did you run to trigger this issue? If you can provide a [Minimal, Complete, and Verifiable example](http://stackoverflow.com/help/mcve) this will help us understand the issue. + render: shell + validations: + required: true + - type: textarea + id: expected-behaviour + attributes: + label: Expected behaviour + description: What did you expect to occur after running these commands? + validations: + required: true + - type: textarea + id: actual-behaviour + attributes: + label: Actual behaviour + description: What actually happened instead? + validations: + required: true + - type: textarea + id: repository + attributes: + label: Repository + description: If the problem was occurring with a specific repository, can you provide the URL to that repository to help us with testing? \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000000000..ec4bb386bcf8a4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: false \ No newline at end of file From 4d8ae204c8e04e12687fe814aa8081bdffa45859 Mon Sep 17 00:00:00 2001 From: Philip Oakley Date: Fri, 22 Dec 2017 17:15:50 +0000 Subject: [PATCH 224/229] Modify the GitHub Pull Request template (to reflect Git for Windows) Git for Windows accepts pull requests; Core Git does not. Therefore we need to adjust the template (because it only matches core Git's project management style, not ours). Also: direct Git for Windows enhancements to their contributions page, space out the text for easy reading, and clarify that the mailing list is plain text, not HTML. Signed-off-by: Philip Oakley Signed-off-by: Johannes Schindelin --- .github/PULL_REQUEST_TEMPLATE.md | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 37654cdfd7abcf..7baf31f2c471ec 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,19 @@ -Thanks for taking the time to contribute to Git! Please be advised that the -Git community does not use github.com for their contributions. Instead, we use -a mailing list (git@vger.kernel.org) for code submissions, code reviews, and -bug reports. Nevertheless, you can use GitGitGadget (https://gitgitgadget.github.io/) +Thanks for taking the time to contribute to Git! + +Those seeking to contribute to the Git for Windows fork should see +http://gitforwindows.org/#contribute on how to contribute Windows specific +enhancements. + +If your contribution is for the core Git functions and documentation +please be aware that the Git community does not use the github.com issues +or pull request mechanism for their contributions. + +Instead, we use the Git mailing list (git@vger.kernel.org) for code and +documentation submissions, code reviews, and bug reports. The +mailing list is plain text only (anything with HTML is sent directly +to the spam folder). + +Nevertheless, you can use GitGitGadget (https://gitgitgadget.github.io/) to conveniently send your Pull Requests commits to our mailing list. For a single-commit pull request, please *leave the pull request description From a33e9d830383d6981c8626e975bebd82ecbadfb1 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 23 Aug 2019 14:14:42 +0200 Subject: [PATCH 225/229] SECURITY.md: document Git for Windows' policies This is the recommended way on GitHub to describe policies revolving around security issues and about supported versions. Helped-by: Sven Strickroth Signed-off-by: Johannes Schindelin --- SECURITY.md | 56 +++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index c720c2ae7f9580..42b6d458bfd557 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -28,24 +28,38 @@ Examples for details to include: ## Supported Versions -There are no official "Long Term Support" versions in Git. -Instead, the maintenance track (i.e. the versions based on the -most recently published feature release, also known as ".0" -version) sees occasional updates with bug fixes. - -Fixes to vulnerabilities are made for the maintenance track for -the latest feature release and merged up to the in-development -branches. The Git project makes no formal guarantee for any -older maintenance tracks to receive updates. In practice, -though, critical vulnerability fixes are applied not only to the -most recent track, but to at least a couple more maintenance -tracks. - -This is typically done by making the fix on the oldest and still -relevant maintenance track, and merging it upwards to newer and -newer maintenance tracks. - -For example, v2.24.1 was released to address a couple of -[CVEs](https://cve.mitre.org/), and at the same time v2.14.6, -v2.15.4, v2.16.6, v2.17.3, v2.18.2, v2.19.3, v2.20.2, v2.21.1, -v2.22.2 and v2.23.1 were released. +Git for Windows is a "friendly fork" of [Git](https://git-scm.com/), i.e. changes in Git for Windows are frequently contributed back, and Git for Windows' release cycle closely following Git's. + +While Git maintains several release trains (when v2.19.1 was released, there were updates to v2.14.x-v2.18.x, too, for example), Git for Windows follows only the latest Git release. For example, there is no Git for Windows release corresponding to Git v2.16.5 (which was released after v2.19.0). + +One exception is [MinGit for Windows](https://gitforwindows.org/mingit) (a minimal subset of Git for Windows, intended for bundling with third-party applications that do not need any interactive commands nor support for `git svn`): critical security fixes are backported to the v2.11.x, v2.14.x, v2.19.x, v2.21.x and v2.23.x release trains. + +## Version number scheme + +The Git for Windows versions reflect the Git version on which they are based. For example, Git for Windows v2.21.0 is based on Git v2.21.0. + +As Git for Windows bundles more than just Git (such as Bash, OpenSSL, OpenSSH, GNU Privacy Guard), sometimes there are interim releases without corresponding Git releases. In these cases, Git for Windows appends a number in parentheses, starting with the number 2, then 3, etc. For example, both Git for Windows v2.17.1 and v2.17.1(2) were based on Git v2.17.1, but the latter included updates for Git Credential Manager and Git LFS, fixing critical regressions. + +## Tag naming scheme + +Every Git for Windows version is tagged using a name that starts with the Git version on which it is based, with the suffix `.windows.` appended. For example, Git for Windows v2.17.1' source code is tagged as [`v2.17.1.windows.1`](https://github.com/git-for-windows/git/releases/tag/v2.17.1.windows.1) (the patch level is always at least 1, given that Git for Windows always has patches on top of Git). Likewise, Git for Windows v2.17.1(2)' source code is tagged as [`v2.17.1.windows.2`](https://github.com/git-for-windows/git/releases/tag/v2.17.1.windows.2). + +## Release Candidate (rc) versions + +As a friendly fork of Git (the "upstream" project), Git for Windows is closely corelated to that project. + +Consequently, Git for Windows publishes versions based on Git's release candidates (for upcoming "`.0`" versions, see [Git's release schedule](https://tinyurl.com/gitCal)). These versions end in `-rc`, starting with `-rc0` for a very early preview of what is to come, and as with regular versions, Git for Windows tries to follow Git's releases as quickly as possible. + +Note: there is currently a bug in the "Check daily for updates" code, where it mistakes the final version as a downgrade from release candidates. Example: if you installed Git for Windows v2.23.0-rc3 and enabled the auto-updater, it would ask you whether you want to "downgrade" to v2.23.0 when that version was available. + +[All releases](https://github.com/git-for-windows/git/releases/), including release candidates, are listed via a link at the footer of the [Git for Windows](https://gitforwindows.org/) home page. + +## Snapshot versions ('nightly builds') + +Git for Windows also provides snapshots (these are not releases) of the current development as per git-for-Windows/git's `master` branch at the [Snapshots](https://gitforwindows.org/git-snapshots/) page. This link is also listed in the footer of the [Git for Windows](https://gitforwindows.org/) home page. + +Note: even if those builds are not exactly "nightly", they are sometimes referred to as "nightly builds" to keep with other projects' nomenclature. + +## Following upstream's developments + +The [gitforwindows/git repository](https://github.com/git-for-windows/git) also provides the `shears/*` branches. The `shears/*` branches reflect Git for Windows' patches, rebased onto the upstream integration branches, [updated (mostly) via automated CI builds](https://dev.azure.com/git-for-windows/git/_build?definitionId=25). From 881842c3e0aceff1c9935a3add6a289131ed128b Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 6 Feb 2024 18:45:35 +0100 Subject: [PATCH 226/229] dependabot: help keeping GitHub Actions versions up to date See https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot#enabling-dependabot-version-updates-for-actions for details. Signed-off-by: Johannes Schindelin --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000000000..22d5376407abf1 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file +# especially +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot#enabling-dependabot-version-updates-for-actions + +version: 2 +updates: + - package-ecosystem: "github-actions" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "weekly" From 1e79291980eb781a6ebf0024b9ef6cd11bfe626e Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Fri, 12 Jun 2026 11:33:40 +0200 Subject: [PATCH 227/229] ci: only run the expensive tests in the Windows tests for now Upstream Git does not test their tags with the expensive set of tests, so a couple of them seem quite broken for now, even so much as hanging indefinitely. It is outside of the responsibility of the Git for Windows project to fix upstream's own tests for platforms other than Windows, so let's not exercise them. Signed-off-by: Johannes Schindelin --- ci/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ci/lib.sh b/ci/lib.sh index ca80ea7bde11e1..3aa317b74f485f 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -336,6 +336,7 @@ export SKIP_DASHED_BUILT_INS=YesPlease # enable "expensive" tests for PR events. # In order to catch bugs introduced at integration time by mismerges, # enable the long tests for pushes to the integration branches as well. +test -z "$MSYSTEM" || case "$CI_EVENT,$CI_BRANCH" in pull_request,*|push,*next*|push,*master*|push,*main*|push,*maint*) export GIT_TEST_LONG=${GIT_TEST_LONG:-true} From 084acd9a072b146bef572488281a11d90cb46588 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Tue, 29 Sep 2026 15:40:53 +0200 Subject: [PATCH 228/229] mingw: let `access()` handle `nul` whether lower-case or not The special-casing of `NUL` in `mingw_access()` was added in 9160068ac6ec5 (msvc: avoid calling `access("NUL", flags)`, 2021-04-16) because the Universal C Runtime's ("UCRT") `_waccess()` rejects `NUL` as an invalid argument. With the switch of GIt for Windows from the MSVCRT-based MINGW64 build to the UCRT-based UCRT64 build in 2.56.0, any spelling other than lowercase `nul` now reaches `_waccess()` and fails. Windows device names are case-insensitive, and `NUL` is the canonical spelling, but even `Nul` is allowed, so a case-insensitive comparison is needed. Note that we cannot use `fspathcmp()` here, as we would usually do: This function is in the code path when reading the config, and `core.ignoreCase` would therefore not be set correctly yet e.g. when running under `GIT_CONFIG_GLOBAL=NUL`. This fixes https://github.com/git-for-windows/git/issues/6449 Signed-off-by: Johannes Schindelin --- compat/mingw.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compat/mingw.c b/compat/mingw.c index fd014eef498e55..9c5a5140790161 100644 --- a/compat/mingw.c +++ b/compat/mingw.c @@ -1171,7 +1171,11 @@ ssize_t mingw_write(int fd, const void *buf, size_t len) int mingw_access(const char *filename, int mode) { wchar_t wfilename[MAX_LONG_PATH]; - if (!strcmp("nul", filename) || !strcmp("/dev/null", filename)) + /* + * Cannot use fspathcmp() here, this function is called _while reading + * the config_. + */ + if (!strcasecmp("nul", filename) || !strcmp("/dev/null", filename)) return 0; if (xutftowcs_long_path(wfilename, filename) < 0) return -1; From 7af13ba6fd2472ac067494e4a9f17b76069cfe0a Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Mon, 5 Oct 2026 09:16:27 +0000 Subject: [PATCH 229/229] ci: bump actions/setup-python from 6 to 7 This bumps `actions/setup-python from version 6 to 7. The main change is that this is now an ECMAScript Module, which bumps the minimum requirements for GitHub Enterprise Server setups enough to warrant a full major version bump. Release notes: https://github.com/actions/setup-python/releases Commits: https://github.com/actions/setup-python/compare/v6...v7 Originally-authored-by: dependabot[bot] Signed-off-by: Johannes Schindelin --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71cabd013fb4f1..40cc3690ef7ee2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -290,7 +290,7 @@ jobs: cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 - name: Set up dependencies shell: pwsh run: pip install meson ninja @@ -318,7 +318,7 @@ jobs: cancel-in-progress: ${{ needs.ci-config.outputs.skip_concurrent == 'yes' }} steps: - uses: actions/checkout@v7 - - uses: actions/setup-python@v6 + - uses: actions/setup-python@v7 - name: Set up dependencies shell: pwsh run: pip install meson ninja