[v2,0/7] Show progress when downloading from bundle URIs

Message

Toon Claes Feb. 19, 2025, 2:30 p.m. UTC

When a user clones a repository, they see what's happening in the
messages like "Enumerating objects" and "Receiving objects". But when
a user clones a repository that uses bundle URIs they see:

    Cloning into 'repo.git'

And then they have to wait until all bundles are downloaded before they
see any other message. When the bundles are large, this can take a lot
of time and the user might consider the process hangs and they kill it.

This patch series introduces progress displayed to the user while
bundles are downloaded. The full output of a clone using bundle URIs
will look something like:

    Cloning into 'repo.git'...
    Downloading via HTTP:  21% (351812809/1620086598), 315.34 MiB | 49.84 MiB/s
    Downloading via HTTP:  77% (1247493865/1620086598), 1.15 GiB | 34.31 MiB/s
    Downloading via HTTP: 100% (1620086598/1620086598), 1.50 GiB | 37.97 MiB/s, done.
    remote: Enumerating objects: 1322255, done.
    remote: Counting objects: 100% (611708/611708), done.
    remote: Total 1322255 (delta 611708), reused 611708 (delta 611708), pack-reused 710547
    Receiving objects: 100% (1322255/1322255), 539.66 MiB | 31.57 MiB/s, done.
    etc...

In this version I'm adding two commits from Peff. And I'm adding one
commit on top to fix the failure in t5558. I'd like to get some feedback
on that commit in particular.

---
Changes in v2:
- Added commit "http: silence stderr when progress is enabled"
- Added commit "progress: allow pure-throughput progress meters" from
  https://lore.kernel.org/git/[e-mail]/
- Added commit "http: turn off curl signals" from
  https://lore.kernel.org/git/[e-mail]/
- Cleanup both CURLOPT_NOPROGRESS and CURLOPT_NOPROGRESS.
- Remove unneeded fflush().
- Link to v1: https://lore.kernel.org/git/[e-mail]/

Signed-off-by: Toon Claes <[e-mail]>

---
Jeff King (2):
      progress: allow pure-throughput progress meters
      http: turn off curl signals

Toon Claes (5):
      progress: add function to set total
      http: add the ability to log progress
      remote-curl: optionally show progress for HTTP get
      bundle-uri: enable git-remote-https progress
      http: silence stderr when progress is enabled

 bundle-uri.c                |  2 +-
 http.c                      | 39 ++++++++++++++++++++++++++++++++
 http.h                      |  5 +++++
 progress.c                  | 40 ++++++++++++++++++++++++---------
 progress.h                  |  2 ++
 remote-curl.c               |  8 ++++++-
 t/helper/test-progress.c    |  5 +++++
 t/t0500-progress-display.sh | 55 +++++++++++++++++++++++++++++++++++++++++++++
 t/t5557-http-get.sh         | 15 +++++++++++++
 9 files changed, 159 insertions(+), 12 deletions(-)
---

Range-diff versus v1:

1:  3eb1958b8b ! 1:  7acc83d92b progress: add function to set total
    @@ Commit message
         Signed-off-by: Toon Claes <[e-mail]>
     
      ## progress.c ##
    -@@ progress.c: static struct progress *start_progress_delay(const char *title, uint64_t total,
    +@@ progress.c: static struct progress *start_progress_delay(struct repository *r,
      	return progress;
      }
      
    @@ progress.h: void progress_test_force_update(void);
      void display_throughput(struct progress *progress, uint64_t total);
      void display_progress(struct progress *progress, uint64_t n);
     +void progress_set_total(struct progress *progress, uint64_t total);
    - struct progress *start_progress(const char *title, uint64_t total);
    - struct progress *start_sparse_progress(const char *title, uint64_t total);
    - struct progress *start_delayed_progress(const char *title, uint64_t total);
    + struct progress *start_progress(struct repository *r,
    + 				const char *title, uint64_t total);
    + struct progress *start_sparse_progress(struct repository *r,
     
      ## t/helper/test-progress.c ##
     @@ t/helper/test-progress.c: int cmd__progress(int argc, const char **argv)
      			if (*end != '\0')
    - 				die("invalid input: '%s'\n", line.buf);
    + 				die("invalid input: '%s'", line.buf);
      			display_progress(progress, item_count);
     +		} else if (skip_prefix(line.buf, "total ", (const char **) &end)) {
     +			uint64_t total = strtoull(end, &end, 10);
-:  ---------- > 2:  7a81cda492 progress: allow pure-throughput progress meters
-:  ---------- > 3:  86a9120539 http: turn off curl signals
2:  b74ce999a1 ! 4:  3a73d68448 http: add the ability to log progress
    @@ http.c: static void http_opt_request_remainder(CURL *curl, off_t pos)
      #define HTTP_REQUEST_FILE	1
      
     +static int http_progress_callback(void *clientp, curl_off_t dltotal,
    -+				  curl_off_t dlnow, curl_off_t ultotal,
    -+				  curl_off_t ulnow)
    ++				  curl_off_t dlnow, curl_off_t ultotal UNUSED,
    ++				  curl_off_t ulnow UNUSED)
     +{
     +	struct progress *progress = clientp;
     +
    @@ http.c: static int http_request(const char *url,
      	    http_follow_config == HTTP_FOLLOW_INITIAL)
      		curl_easy_setopt(slot->curl, CURLOPT_FOLLOWLOCATION, 1);
     +	if (options && options->progress) {
    -+		progress = start_progress(_("Downloading via HTTP"), 0);
    ++		progress = start_progress(the_repository, _("Downloading via HTTP"), 0);
     +
     +		curl_easy_setopt(slot->curl, CURLOPT_NOPROGRESS, 0L);
     +		curl_easy_setopt(slot->curl, CURLOPT_XFERINFODATA, progress);
    @@ http.c: static int http_request(const char *url,
      	ret = run_one_slot(slot, &results);
      
     +	if (progress) {
    ++		curl_easy_setopt(slot->curl, CURLOPT_NOPROGRESS, 1L);
     +		curl_easy_setopt(slot->curl, CURLOPT_XFERINFODATA, NULL);
    ++		curl_easy_setopt(slot->curl, CURLOPT_XFERINFOFUNCTION, NULL);
     +		stop_progress(&progress);
     +	}
     +
3:  5b5e7e86c5 ! 5:  01b2a0042f remote-curl: optionally show progress for HTTP get
    @@ t/t5557-http-get.sh: test_expect_success 'get by URL: 200' '
     +	EOF
     +
     +	git remote-http $url <input 2>err &&
    -+        test_grep "^Downloading via HTTP: 100%" err
    ++	test_grep "^Downloading via HTTP: 100%" err
     +'
     +
      test_done
4:  98ef4dfade ! 6:  3df90aa17b bundle-uri: enable git-remote-https progress
    @@ bundle-uri.c: static int download_https_uri_to_file(const char *file, const char
      	}
      
     +	fprintf(child_in, "option progress true\n");
    -+	fflush(child_in);
    -+
      	fprintf(child_in, "get %s %s\n\n", uri, file);
      
      cleanup:
-:  ---------- > 7:  c30c3bf4fe http: silence stderr when progress is enabled


---

base-commit: a554262210b4a2ee6fa2d594e1f09f5830888c56
change-id: 20250219-toon-bundleuri-progress-3d902efeba06

Thanks
--
Toon