diff --git a/builtin/fetch.c b/builtin/fetch.c index 1e0854c2dbe549..46151a8edb1e25 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -1401,6 +1401,7 @@ static int check_exist_and_connected(struct ref *ref_map) { struct ref *rm = ref_map; struct check_connected_options opt = CHECK_CONNECTED_INIT; + enum odb_has_object_flags flags = ODB_HAS_OBJECT_RECHECK_PACKED; struct ref *r; /* @@ -1420,14 +1421,17 @@ static int check_exist_and_connected(struct ref *ref_map) if (refetch) return -1; - /* * check_connected() allows objects to merely be promised, but - * we need all direct targets to exist. + * we need all direct targets to exist. A virtualized object store + * may need to fetch those targets through its object providers. */ + if (gvfs_virtualize_objects(the_repository)) + flags |= ODB_HAS_OBJECT_FETCH_PROMISOR; + for (r = rm; r; r = r->next) { if (!odb_has_object(the_repository->objects, &r->old_oid, - ODB_HAS_OBJECT_RECHECK_PACKED)) + flags)) return -1; } diff --git a/compat/lazyload-curl.c b/compat/lazyload-curl.c index a6a3f7e3a7aeaa..c361eab85cd1fb 100644 --- a/compat/lazyload-curl.c +++ b/compat/lazyload-curl.c @@ -175,6 +175,20 @@ static curl_easy_setopt_pointer_type curl_easy_setopt_pointer_func; typedef CURLcode (*curl_easy_setopt_off_t_type)(CURL *curl, CURLoption opt, curl_off_t value); static curl_easy_setopt_off_t_type curl_easy_setopt_off_t_func; +typedef CURLU *(*curl_url_type)(void); +static curl_url_type curl_url_func; + +typedef void (*curl_url_cleanup_type)(CURLU *u); +static curl_url_cleanup_type curl_url_cleanup_func; + +typedef CURLUcode (*curl_url_get_type)(const CURLU *u, CURLUPart what, + char **part, unsigned int flags); +static curl_url_get_type curl_url_get_func; + +typedef CURLUcode (*curl_url_set_type)(CURLU *u, CURLUPart what, + const char *part, unsigned int flags); +static curl_url_set_type curl_url_set_func; + static char ssl_backend[64]; static void lazy_load_curl(void) @@ -231,6 +245,11 @@ static void lazy_load_curl(void) curl_easy_setopt_long_func = (curl_easy_setopt_long_type)curl_easy_setopt_func; curl_easy_setopt_pointer_func = (curl_easy_setopt_pointer_type)curl_easy_setopt_func; curl_easy_setopt_off_t_func = (curl_easy_setopt_off_t_type)curl_easy_setopt_func; + + curl_url_func = (curl_url_type)load_function(libcurl, "curl_url"); + curl_url_cleanup_func = (curl_url_cleanup_type)load_function(libcurl, "curl_url_cleanup"); + curl_url_get_func = (curl_url_get_type)load_function(libcurl, "curl_url_get"); + curl_url_set_func = (curl_url_set_type)load_function(libcurl, "curl_url_set"); } struct curl_version_info_data *curl_version_info(CURLversion version) @@ -426,3 +445,25 @@ CURLcode curl_easy_setopt(CURL *curl, CURLoption opt, ...) va_end(ap); return res; } + +CURLU *curl_url(void) +{ + return curl_url_func(); +} + +void curl_url_cleanup(CURLU *u) +{ + curl_url_cleanup_func(u); +} + +CURLUcode curl_url_get(const CURLU *u, CURLUPart what, + char **part, unsigned int flags) +{ + return curl_url_get_func(u, what, part, flags); +} + +CURLUcode curl_url_set(CURLU *u, CURLUPart what, + const char *part, unsigned int flags) +{ + return curl_url_set_func(u, what, part, flags); +} diff --git a/git-curl-compat.h b/git-curl-compat.h index 9002b7107325f6..c4bf7e6b04ebb3 100644 --- a/git-curl-compat.h +++ b/git-curl-compat.h @@ -28,6 +28,13 @@ * introduced, oldest first, in the official version of cURL library. */ +/** + * curl_url() interface added in 7.62.0 (October 2018) + */ +#if LIBCURL_VERSION_NUM >= 0x073e00 +#define GIT_CURL_HAVE_CURL_URL +#endif + /** * Versions before curl 7.66.0 (September 2019) required manually setting the * transfer-encoding for a streaming POST; after that this is handled diff --git a/http.c b/http.c index c3fd4f14278ef9..67dcf67e8372f4 100644 --- a/http.c +++ b/http.c @@ -2401,6 +2401,45 @@ static int http_request(const char *url, return ret; } +#ifndef GIT_CURL_HAVE_CURL_URL +#define strip_url_credential(in) NULL +#else +static char *strip_url_credential(const char *in) +{ + char *ret = NULL; + CURLU *url; + + url = curl_url(); + if (!url) + goto out; + + if (curl_url_set(url, CURLUPART_URL, in, 0)) + goto out; + + curl_url_set(url, CURLUPART_USER, NULL, 0); + curl_url_set(url, CURLUPART_PASSWORD, NULL, 0); + curl_url_get(url, CURLUPART_URL, &ret, 0); + +out: + curl_url_cleanup(url); + return ret; +} +#endif + +static int match_effective_url(const char *asked, const char *got) +{ + char *stripped; + int ret; + + if (!strcmp(asked, got)) + return 1; + + stripped = strip_url_credential(asked); + ret = stripped && !strcmp(stripped, got); + curl_free(stripped); + return ret; +} + /* * Update the "base" url to a more appropriate value, as deduced by * redirects seen when requesting a URL starting with "url". @@ -2433,7 +2472,7 @@ static int update_url_from_redirect(struct strbuf *base, const char *tail; size_t new_len; - if (!strcmp(asked, got->buf)) + if (match_effective_url(asked, got->buf)) return 0; if (!skip_prefix(asked, base->buf, &tail)) diff --git a/t/t0410/read-object-all b/t/t0410/read-object-all new file mode 100755 index 00000000000000..2b18f467dfc3c3 --- /dev/null +++ b/t/t0410/read-object-all @@ -0,0 +1,114 @@ +#!/usr/bin/perl +# +# Example implementation of the Git read-object protocol version 1 that +# serves objects of any type (not just blobs) from a host repository. +# See Documentation/technical/read-object-protocol.txt +# +# The host repository is taken from the READ_OBJECT_DIR environment +# variable so the same hook can be reused by different tests. It must +# point at the ".git" directory (or bare repository) that holds the +# objects to be served. +# + +use strict; +use warnings; + +my $DIR = $ENV{"READ_OBJECT_DIR"} || "../.git/"; + +sub packet_bin_read { + my $buffer; + my $bytes_read = read STDIN, $buffer, 4; + if ( $bytes_read == 0 ) { + + # EOF - Git stopped talking to us! + exit(); + } + elsif ( $bytes_read != 4 ) { + die "invalid packet: '$buffer'"; + } + my $pkt_size = hex($buffer); + if ( $pkt_size == 0 ) { + return ( 1, "" ); + } + elsif ( $pkt_size > 4 ) { + my $content_size = $pkt_size - 4; + $bytes_read = read STDIN, $buffer, $content_size; + if ( $bytes_read != $content_size ) { + die "invalid packet ($content_size bytes expected; " . + "$bytes_read bytes read)"; + } + return ( 0, $buffer ); + } + else { + die "invalid packet size: $pkt_size"; + } +} + +sub packet_txt_read { + my ( $res, $buf ) = packet_bin_read(); + unless ( $buf =~ s/\n$// ) { + die "A non-binary line MUST be terminated by an LF."; + } + return ( $res, $buf ); +} + +sub packet_bin_write { + my $buf = shift; + print STDOUT sprintf( "%04x", length($buf) + 4 ); + print STDOUT $buf; + STDOUT->flush(); +} + +sub packet_txt_write { + packet_bin_write( $_[0] . "\n" ); +} + +sub packet_flush { + print STDOUT sprintf( "%04x", 0 ); + STDOUT->flush(); +} + +( packet_txt_read() eq ( 0, "git-read-object-client" ) ) || + die "bad initialize"; +( packet_txt_read() eq ( 0, "version=1" ) ) || die "bad version"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad version end"; + +packet_txt_write("git-read-object-server"); +packet_txt_write("version=1"); +packet_flush(); + +( packet_txt_read() eq ( 0, "capability=get" ) ) || die "bad capability"; +( packet_bin_read() eq ( 1, "" ) ) || die "bad capability end"; + +packet_txt_write("capability=get"); +packet_flush(); + +while (1) { + my ($command) = packet_txt_read() =~ /^command=([^=]+)$/; + + if ( $command eq "get" ) { + my ($sha1) = packet_txt_read() =~ /^sha1=([0-9a-f]{40,64})$/; + packet_bin_read(); + + my $type = `git --git-dir="$DIR" cat-file -t $sha1 2>/dev/null`; + chomp $type; + if ( $type eq "" ) { + packet_txt_write("status=error"); + packet_flush(); + next; + } + + system ('git --git-dir="' . $DIR . '" cat-file ' . + $type . ' ' . $sha1 . + ' | git -c core.virtualizeobjects=false hash-object ' . + '-t ' . $type . ' -w --stdin >/dev/null 2>&1'); + packet_txt_write(($?) ? "status=error" : "status=success"); + packet_flush(); + + open my $log, '>>.git/read-object-hook.log'; + print $log "Read object $sha1, exit code $?\n"; + close $log; + } else { + die "bad command '$command'"; + } +} diff --git a/t/t0499-read-object.sh b/t/t0499-read-object.sh index 5ae8ca2676f3b4..6c7c56b83c2fed 100755 --- a/t/t0499-read-object.sh +++ b/t/t0499-read-object.sh @@ -85,4 +85,44 @@ test_expect_success 'no-fetch commit lookups still report corruption' ' done ' +test_expect_success PERL_TEST_HELPERS 'setup fetch with a virtualized ref tip' ' + git init -b main fetch-server && + git -C fetch-server commit --allow-empty -m only && + server_tip=$(git -C fetch-server rev-parse HEAD) && + server_odb="$(pwd)/fetch-server/.git" && + + git init -b main fetch-client && + mkdir -p fetch-client/.git/hooks && + sed "1s|/usr/bin/perl|$PERL_PATH|" \ + <$TEST_DIRECTORY/t0410/read-object-all \ + >fetch-client/.git/hooks/read-object && + chmod +x fetch-client/.git/hooks/read-object && + git -C fetch-client config core.virtualizeobjects true +' + +# The ref tip is not in the client object store and is newer than any +# pack a GVFS cache server has published. The fetch existence check must +# allow object acquisition in a virtualized repository so that the +# read-object hook supplies the tip without a fetch-pack exchange. +test_expect_success PERL_TEST_HELPERS 'read-object hook runs for missing tip' ' + test_path_is_missing fetch-client/.git/read-object-hook.log && + + READ_OBJECT_DIR="$server_odb" \ + GIT_TRACE2_EVENT="$PWD/fetch.trace" \ + git -C fetch-client fetch "$(pwd)/fetch-server" \ + +refs/heads/main:refs/remotes/server/main && + + echo "$server_tip" >expect && + git -C fetch-client rev-parse refs/remotes/server/main >actual && + test_cmp expect actual && + + test_grep ! "\"label\":\"fetch_refs\"" fetch.trace && + + find fetch-client/.git/objects/pack -name "*.pack" >packs && + test_must_be_empty packs && + + test_path_is_file fetch-client/.git/read-object-hook.log && + test_grep "$server_tip" fetch-client/.git/read-object-hook.log +' + test_done