@@ -447,6 +447,139 @@ static int download_bundle_to_file(struct remote_bundle_info *bundle, void *data
return 0;
}
+struct bundles_for_sorting {
+ struct remote_bundle_info **items;
+ size_t alloc;
+ size_t nr;
+};
+
+static int append_bundle(struct remote_bundle_info *bundle, void *data)
+{
+ struct bundles_for_sorting *list = data;
+ list->items[list->nr++] = bundle;
+ return 0;
+}
+
+/**
+ * For use in QSORT() to get a list sorted by creationToken
+ * in decreasing order.
+ */
+static int compare_creation_token_decreasing(const void *va, const void *vb)
+{
+ const struct remote_bundle_info * const *a = va;
+ const struct remote_bundle_info * const *b = vb;
+
+ if ((*a)->creationToken > (*b)->creationToken)
+ return -1;
+ if ((*a)->creationToken < (*b)->creationToken)
+ return 1;
+ return 0;
+}
+
+static int fetch_bundles_by_token(struct repository *r,
+ struct bundle_list *list)
+{
+ int cur;
+ int move_direction = 0;
+ struct bundle_list_context ctx = {
+ .r = r,
+ .list = list,
+ .mode = list->mode,
+ };
+ struct bundles_for_sorting bundles = {
+ .alloc = hashmap_get_size(&list->bundles),
+ };
+
+ ALLOC_ARRAY(bundles.items, bundles.alloc);
+
+ for_all_bundles_in_list(list, append_bundle, &bundles);
+
+ QSORT(bundles.items, bundles.nr, compare_creation_token_decreasing);
+
+ /*
+ * Attempt to download and unbundle the minimum number of bundles by
+ * creationToken in decreasing order. If we fail to unbundle (after
+ * a successful download) then move to the next non-downloaded bundle
+ * and attempt downloading. Once we succeed in applying a bundle,
+ * move to the previous unapplied bundle and attempt to unbundle it
+ * again.
+ *
+ * In the case of a fresh clone, we will likely download all of the
+ * bundles before successfully unbundling the oldest one, then the
+ * rest of the bundles unbundle successfully in increasing order
+ * of creationToken.
+ *
+ * If there are existing objects, then this process may terminate
+ * early when all required commits from "new" bundles exist in the
+ * repo's object store.
+ */
+ cur = 0;
+ while (cur >= 0 && cur < bundles.nr) {
+ struct remote_bundle_info *bundle = bundles.items[cur];
+ if (!bundle->file) {
+ /*
+ * Not downloaded yet. Try downloading.
+ *
+ * Note that bundle->file is non-NULL if a download
+ * was attempted, even if it failed to download.
+ */
+ if (fetch_bundle_uri_internal(ctx.r, bundle, ctx.depth + 1, ctx.list)) {
+ /* Mark as unbundled so we do not retry. */
+ bundle->unbundled = 1;
+
+ /* Try looking deeper in the list. */
+ move_direction = 1;
+ goto move;
+ }
+
+ /* We expect bundles when using creationTokens. */
+ if (!is_bundle(bundle->file, 1)) {
+ warning(_("file downloaded from '%s' is not a bundle"),
+ bundle->uri);
+ break;
+ }
+ }
+
+ if (bundle->file && !bundle->unbundled) {
+ /*
+ * This was downloaded, but not successfully
+ * unbundled. Try unbundling again.
+ */
+ if (unbundle_from_file(ctx.r, bundle->file)) {
+ /* Try looking deeper in the list. */
+ move_direction = 1;
+ } else {
+ /*
+ * Succeeded in unbundle. Retry bundles
+ * that previously failed to unbundle.
+ */
+ move_direction = -1;
+ bundle->unbundled = 1;
+ }
+ }
+
+ /*
+ * Else case: downloaded and unbundled successfully.
+ * Skip this by moving in the same direction as the
+ * previous step.
+ */
+
+move:
+ /* Move in the specified direction and repeat. */
+ cur += move_direction;
+ }
+
+ free(bundles.items);
+
+ /*
+ * We succeed if the loop terminates because 'cur' drops below
+ * zero. The other case is that we terminate because 'cur'
+ * reaches the end of the list, so we have a failure no matter
+ * which bundles we apply from the list.
+ */
+ return cur >= 0;
+}
+
static int download_bundle_list(struct repository *r,
struct bundle_list *local_list,
struct bundle_list *global_list,
@@ -484,7 +617,15 @@ static int fetch_bundle_list_in_config_format(struct repository *r,
goto cleanup;
}
- if ((result = download_bundle_list(r, &list_from_bundle,
+ /*
+ * If this list uses the creationToken heuristic, then the URIs
+ * it advertises are expected to be bundles, not nested lists.
+ * We can drop 'global_list' and 'depth'.
+ */
+ if (list_from_bundle.heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN) {
+ result = fetch_bundles_by_token(r, &list_from_bundle);
+ global_list->heuristic = BUNDLE_HEURISTIC_CREATIONTOKEN;
+ } else if ((result = download_bundle_list(r, &list_from_bundle,
global_list, depth)))
goto cleanup;
@@ -626,6 +767,14 @@ int fetch_bundle_list(struct repository *r, struct bundle_list *list)
int result;
struct bundle_list global_list;
+ /*
+ * If the creationToken heuristic is used, then the URIs
+ * advertised by 'list' are not nested lists and instead
+ * direct bundles. We do not need to use global_list.
+ */
+ if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
+ return fetch_bundles_by_token(r, list);
+
init_bundle_list(&global_list);
/* If a bundle is added to this global list, then it is required. */
@@ -634,7 +783,10 @@ int fetch_bundle_list(struct repository *r, struct bundle_list *list)
if ((result = download_bundle_list(r, list, &global_list, 0)))
goto cleanup;
- result = unbundle_all_bundles(r, &global_list);
+ if (list->heuristic == BUNDLE_HEURISTIC_CREATIONTOKEN)
+ result = fetch_bundles_by_token(r, list);
+ else
+ result = unbundle_all_bundles(r, &global_list);
cleanup:
for_all_bundles_in_list(&global_list, unlink_bundle, NULL);
@@ -401,17 +401,43 @@ test_expect_success 'clone bundle list (http, creationToken)' '
git -C clone-list-http-2 cat-file --batch-check <oids &&
cat >expect <<-EOF &&
- $HTTPD_URL/bundle-1.bundle
- $HTTPD_URL/bundle-2.bundle
- $HTTPD_URL/bundle-3.bundle
+ $HTTPD_URL/bundle-list
$HTTPD_URL/bundle-4.bundle
+ $HTTPD_URL/bundle-3.bundle
+ $HTTPD_URL/bundle-2.bundle
+ $HTTPD_URL/bundle-1.bundle
+ EOF
+
+ test_remote_https_urls <trace-clone.txt >actual &&
+ test_cmp expect actual
+'
+
+test_expect_success 'clone incomplete bundle list (http, creationToken)' '
+ test_when_finished rm -f trace*.txt &&
+
+ cp clone-from/bundle-*.bundle "$HTTPD_DOCUMENT_ROOT_PATH/" &&
+ cat >"$HTTPD_DOCUMENT_ROOT_PATH/bundle-list" <<-EOF &&
+ [bundle]
+ version = 1
+ mode = all
+ heuristic = creationToken
+
+ [bundle "bundle-1"]
+ uri = bundle-1.bundle
+ creationToken = 1
+ EOF
+
+ GIT_TRACE2_EVENT=$(pwd)/trace-clone.txt \
+ git clone --bundle-uri="$HTTPD_URL/bundle-list" \
+ --single-branch --branch=base --no-tags \
+ "$HTTPD_URL/smart/fetch.git" clone-token-http &&
+
+ cat >expect <<-EOF &&
$HTTPD_URL/bundle-list
+ $HTTPD_URL/bundle-1.bundle
EOF
- # Since the creationToken heuristic is not yet understood by the
- # client, the order cannot be verified at this moment. Sort the
- # list for consistent results.
- test_remote_https_urls <trace-clone.txt | sort >actual &&
+ test_remote_https_urls <trace-clone.txt >actual &&
test_cmp expect actual
'
@@ -831,6 +831,52 @@ test_expect_success 'auto-discover multiple bundles from HTTP clone' '
grep -f pattern trace.txt
'
+test_expect_success 'auto-discover multiple bundles from HTTP clone: creationToken heuristic' '
+ test_when_finished rm -rf "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" &&
+ test_when_finished rm -rf clone-heuristic trace*.txt &&
+
+ test_commit -C src newest &&
+ git -C src bundle create "$HTTPD_DOCUMENT_ROOT_PATH/newest.bundle" HEAD~1..HEAD &&
+ git clone --bare --no-local src "$HTTPD_DOCUMENT_ROOT_PATH/repo4.git" &&
+
+ cat >>"$HTTPD_DOCUMENT_ROOT_PATH/repo4.git/config" <<-EOF &&
+ [uploadPack]
+ advertiseBundleURIs = true
+
+ [bundle]
+ version = 1
+ mode = all
+ heuristic = creationToken
+
+ [bundle "everything"]
+ uri = $HTTPD_URL/everything.bundle
+ creationtoken = 1
+
+ [bundle "new"]
+ uri = $HTTPD_URL/new.bundle
+ creationtoken = 2
+
+ [bundle "newest"]
+ uri = $HTTPD_URL/newest.bundle
+ creationtoken = 3
+ EOF
+
+ GIT_TRACE2_EVENT="$(pwd)/trace-clone.txt" \
+ git -c protocol.version=2 \
+ -c transfer.bundleURI=true clone \
+ "$HTTPD_URL/smart/repo4.git" clone-heuristic &&
+
+ cat >expect <<-EOF &&
+ $HTTPD_URL/newest.bundle
+ $HTTPD_URL/new.bundle
+ $HTTPD_URL/everything.bundle
+ EOF
+
+ # We should fetch all bundles in the expected order.
+ test_remote_https_urls <trace-clone.txt >actual &&
+ test_cmp expect actual
+'
+
# DO NOT add non-httpd-specific tests here, because the last part of this
# test script is only executed when httpd is available and enabled.