http-walker.c

   1 #include "cache.h"
   2 #include "commit.h"
   3 #include "pack.h"
   4 #include "walker.h"
   5 #include "http.h"
   6
   7 #define PREV_BUF_SIZE 4096
   8
   9 struct alt_base
  10 {
  11         char *base;
  12         int got_indices;
  13         struct packed_git *packs;
  14         struct alt_base *next;
  15 };
  16
  17 enum object_request_state {
  18         WAITING,
  19         ABORTED,
  20         ACTIVE,
  21         COMPLETE,
  22 };
  23
  24 struct object_request
  25 {
  26         struct walker *walker;
  27         unsigned char sha1[20];
  28         struct alt_base *repo;
  29         char *url;
  30         char filename[PATH_MAX];
  31         char tmpfile[PATH_MAX];
  32         int local;
  33         enum object_request_state state;
  34         CURLcode curl_result;
  35         char errorstr[CURL_ERROR_SIZE];
  36         long http_code;
  37         unsigned char real_sha1[20];
  38         git_SHA_CTX c;
  39         z_stream stream;
  40         int zret;
  41         int rename;
  42         struct active_request_slot *slot;
  43         struct object_request *next;
  44 };
  45
  46 struct alternates_request {
  47         struct walker *walker;
  48         const char *base;
  49         char *url;
  50         struct strbuf *buffer;
  51         struct active_request_slot *slot;
  52         int http_specific;
  53 };
  54
  55 struct walker_data {
  56         const char *url;
  57         int got_alternates;
  58         struct alt_base *alt;
  59 };
  60
  61 static struct object_request *object_queue_head;
  62
  63 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
  64                                void *data)
  65 {
  66         unsigned char expn[4096];
  67         size_t size = eltsize * nmemb;
  68         int posn = 0;
  69         struct object_request *obj_req = (struct object_request *)data;
  70         do {
  71                 ssize_t retval = xwrite(obj_req->local,
  72                                         (char *) ptr + posn, size - posn);
  73                 if (retval < 0)
  74                         return posn;
  75                 posn += retval;
  76         } while (posn < size);
  77
  78         obj_req->stream.avail_in = size;
  79         obj_req->stream.next_in = ptr;
  80         do {
  81                 obj_req->stream.next_out = expn;
  82                 obj_req->stream.avail_out = sizeof(expn);
  83                 obj_req->zret = git_inflate(&obj_req->stream, Z_SYNC_FLUSH);
  84                 git_SHA1_Update(&obj_req->c, expn,
  85                                 sizeof(expn) - obj_req->stream.avail_out);
  86         } while (obj_req->stream.avail_in && obj_req->zret == Z_OK);
  87         data_received++;
  88         return size;
  89 }
  90
  91 static void fetch_alternates(struct walker *walker, const char *base);
  92
  93 static void process_object_response(void *callback_data);
  94
  95 static void start_object_request(struct walker *walker,
  96                                  struct object_request *obj_req)
  97 {
  98         char *hex = sha1_to_hex(obj_req->sha1);
  99         char prevfile[PATH_MAX];
 100         char *url;
 101         char *posn;
 102         int prevlocal;
 103         unsigned char prev_buf[PREV_BUF_SIZE];
 104         ssize_t prev_read = 0;
 105         long prev_posn = 0;
 106         char range[RANGE_HEADER_SIZE];
 107         struct curl_slist *range_header = NULL;
 108         struct active_request_slot *slot;
 109
 110         snprintf(prevfile, sizeof(prevfile), "%s.prev", obj_req->filename);
 111         unlink_or_warn(prevfile);
 112         rename(obj_req->tmpfile, prevfile);
 113         unlink_or_warn(obj_req->tmpfile);
 114
 115         if (obj_req->local != -1)
 116                 error("fd leakage in start: %d", obj_req->local);
 117         obj_req->local = open(obj_req->tmpfile,
 118                               O_WRONLY | O_CREAT | O_EXCL, 0666);
 119         /*
 120          * This could have failed due to the "lazy directory creation";
 121          * try to mkdir the last path component.
 122          */
 123         if (obj_req->local < 0 && errno == ENOENT) {
 124                 char *dir = strrchr(obj_req->tmpfile, '/');
 125                 if (dir) {
 126                         *dir = 0;
 127                         mkdir(obj_req->tmpfile, 0777);
 128                         *dir = '/';
 129                 }
 130                 obj_req->local = open(obj_req->tmpfile,
 131                                       O_WRONLY | O_CREAT | O_EXCL, 0666);
 132         }
 133
 134         if (obj_req->local < 0) {
 135                 obj_req->state = ABORTED;
 136                 error("Couldn't create temporary file %s for %s: %s",
 137                       obj_req->tmpfile, obj_req->filename, strerror(errno));
 138                 return;
 139         }
 140
 141         memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 142
 143         git_inflate_init(&obj_req->stream);
 144
 145         git_SHA1_Init(&obj_req->c);
 146
 147         url = xmalloc(strlen(obj_req->repo->base) + 51);
 148         obj_req->url = xmalloc(strlen(obj_req->repo->base) + 51);
 149         strcpy(url, obj_req->repo->base);
 150         posn = url + strlen(obj_req->repo->base);
 151         strcpy(posn, "/objects/");
 152         posn += 9;
 153         memcpy(posn, hex, 2);
 154         posn += 2;
 155         *(posn++) = '/';
 156         strcpy(posn, hex + 2);
 157         strcpy(obj_req->url, url);
 158
 159         /*
 160          * If a previous temp file is present, process what was already
 161          * fetched.
 162          */
 163         prevlocal = open(prevfile, O_RDONLY);
 164         if (prevlocal != -1) {
 165                 do {
 166                         prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
 167                         if (prev_read>0) {
 168                                 if (fwrite_sha1_file(prev_buf,
 169                                                      1,
 170                                                      prev_read,
 171                                                      obj_req) == prev_read)
 172                                         prev_posn += prev_read;
 173                                 else
 174                                         prev_read = -1;
 175                         }
 176                 } while (prev_read > 0);
 177                 close(prevlocal);
 178         }
 179         unlink_or_warn(prevfile);
 180
 181         /*
 182          * Reset inflate/SHA1 if there was an error reading the previous temp
 183          * file; also rewind to the beginning of the local file.
 184          */
 185         if (prev_read == -1) {
 186                 memset(&obj_req->stream, 0, sizeof(obj_req->stream));
 187                 git_inflate_init(&obj_req->stream);
 188                 git_SHA1_Init(&obj_req->c);
 189                 if (prev_posn>0) {
 190                         prev_posn = 0;
 191                         lseek(obj_req->local, 0, SEEK_SET);
 192                         ftruncate(obj_req->local, 0);
 193                 }
 194         }
 195
 196         slot = get_active_slot();
 197         slot->callback_func = process_object_response;
 198         slot->callback_data = obj_req;
 199         obj_req->slot = slot;
 200
 201         curl_easy_setopt(slot->curl, CURLOPT_FILE, obj_req);
 202         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
 203         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, obj_req->errorstr);
 204         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 205         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 206
 207         /*
 208          * If we have successfully processed data from a previous fetch
 209          * attempt, only fetch the data we don't already have.
 210          */
 211         if (prev_posn>0) {
 212                 if (walker->get_verbosely)
 213                         fprintf(stderr,
 214                                 "Resuming fetch of object %s at byte %ld\n",
 215                                 hex, prev_posn);
 216                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 217                 range_header = curl_slist_append(range_header, range);
 218                 curl_easy_setopt(slot->curl,
 219                                  CURLOPT_HTTPHEADER, range_header);
 220         }
 221
 222         /* Try to get the request started, abort the request on error */
 223         obj_req->state = ACTIVE;
 224         if (!start_active_slot(slot)) {
 225                 obj_req->state = ABORTED;
 226                 obj_req->slot = NULL;
 227                 close(obj_req->local);
 228                 obj_req->local = -1;
 229                 free(obj_req->url);
 230                 return;
 231         }
 232 }
 233
 234 static void finish_object_request(struct object_request *obj_req)
 235 {
 236         struct stat st;
 237
 238         close(obj_req->local);
 239         obj_req->local = -1;
 240
 241         if (obj_req->http_code == 416) {
 242                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
 243         } else if (obj_req->curl_result != CURLE_OK) {
 244                 if (stat(obj_req->tmpfile, &st) == 0)
 245                         if (st.st_size == 0)
 246                                 unlink_or_warn(obj_req->tmpfile);
 247                 return;
 248         }
 249
 250         git_inflate_end(&obj_req->stream);
 251         git_SHA1_Final(obj_req->real_sha1, &obj_req->c);
 252         if (obj_req->zret != Z_STREAM_END) {
 253                 unlink_or_warn(obj_req->tmpfile);
 254                 return;
 255         }
 256         if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 257                 unlink_or_warn(obj_req->tmpfile);
 258                 return;
 259         }
 260         obj_req->rename =
 261                 move_temp_to_file(obj_req->tmpfile, obj_req->filename);
 262
 263         if (obj_req->rename == 0)
 264                 walker_say(obj_req->walker, "got %s\n", sha1_to_hex(obj_req->sha1));
 265 }
 266
 267 static void process_object_response(void *callback_data)
 268 {
 269         struct object_request *obj_req =
 270                 (struct object_request *)callback_data;
 271         struct walker *walker = obj_req->walker;
 272         struct walker_data *data = walker->data;
 273         struct alt_base *alt = data->alt;
 274
 275         obj_req->curl_result = obj_req->slot->curl_result;
 276         obj_req->http_code = obj_req->slot->http_code;
 277         obj_req->slot = NULL;
 278         obj_req->state = COMPLETE;
 279
 280         /* Use alternates if necessary */
 281         if (missing_target(obj_req)) {
 282                 fetch_alternates(walker, alt->base);
 283                 if (obj_req->repo->next != NULL) {
 284                         obj_req->repo =
 285                                 obj_req->repo->next;
 286                         close(obj_req->local);
 287                         obj_req->local = -1;
 288                         start_object_request(walker, obj_req);
 289                         return;
 290                 }
 291         }
 292
 293         finish_object_request(obj_req);
 294 }
 295
 296 static void release_object_request(struct object_request *obj_req)
 297 {
 298         struct object_request *entry = object_queue_head;
 299
 300         if (obj_req->local != -1)
 301                 error("fd leakage in release: %d", obj_req->local);
 302         if (obj_req == object_queue_head) {
 303                 object_queue_head = obj_req->next;
 304         } else {
 305                 while (entry->next != NULL && entry->next != obj_req)
 306                         entry = entry->next;
 307                 if (entry->next == obj_req)
 308                         entry->next = entry->next->next;
 309         }
 310
 311         free(obj_req->url);
 312         free(obj_req);
 313 }
 314
 315 #ifdef USE_CURL_MULTI
 316 static int fill_active_slot(struct walker *walker)
 317 {
 318         struct object_request *obj_req;
 319
 320         for (obj_req = object_queue_head; obj_req; obj_req = obj_req->next) {
 321                 if (obj_req->state == WAITING) {
 322                         if (has_sha1_file(obj_req->sha1))
 323                                 obj_req->state = COMPLETE;
 324                         else {
 325                                 start_object_request(walker, obj_req);
 326                                 return 1;
 327                         }
 328                 }
 329         }
 330         return 0;
 331 }
 332 #endif
 333
 334 static void prefetch(struct walker *walker, unsigned char *sha1)
 335 {
 336         struct object_request *newreq;
 337         struct object_request *tail;
 338         struct walker_data *data = walker->data;
 339         char *filename = sha1_file_name(sha1);
 340
 341         newreq = xmalloc(sizeof(*newreq));
 342         newreq->walker = walker;
 343         hashcpy(newreq->sha1, sha1);
 344         newreq->repo = data->alt;
 345         newreq->url = NULL;
 346         newreq->local = -1;
 347         newreq->state = WAITING;
 348         snprintf(newreq->filename, sizeof(newreq->filename), "%s", filename);
 349         snprintf(newreq->tmpfile, sizeof(newreq->tmpfile),
 350                  "%s.temp", filename);
 351         newreq->slot = NULL;
 352         newreq->next = NULL;
 353
 354         http_is_verbose = walker->get_verbosely;
 355
 356         if (object_queue_head == NULL) {
 357                 object_queue_head = newreq;
 358         } else {
 359                 tail = object_queue_head;
 360                 while (tail->next != NULL)
 361                         tail = tail->next;
 362                 tail->next = newreq;
 363         }
 364
 365 #ifdef USE_CURL_MULTI
 366         fill_active_slots();
 367         step_active_slots();
 368 #endif
 369 }
 370
 371 static void process_alternates_response(void *callback_data)
 372 {
 373         struct alternates_request *alt_req =
 374                 (struct alternates_request *)callback_data;
 375         struct walker *walker = alt_req->walker;
 376         struct walker_data *cdata = walker->data;
 377         struct active_request_slot *slot = alt_req->slot;
 378         struct alt_base *tail = cdata->alt;
 379         const char *base = alt_req->base;
 380         static const char null_byte = '\0';
 381         char *data;
 382         int i = 0;
 383
 384         if (alt_req->http_specific) {
 385                 if (slot->curl_result != CURLE_OK ||
 386                     !alt_req->buffer->len) {
 387
 388                         /* Try reusing the slot to get non-http alternates */
 389                         alt_req->http_specific = 0;
 390                         sprintf(alt_req->url, "%s/objects/info/alternates",
 391                                 base);
 392                         curl_easy_setopt(slot->curl, CURLOPT_URL,
 393                                          alt_req->url);
 394                         active_requests++;
 395                         slot->in_use = 1;
 396                         if (slot->finished != NULL)
 397                                 (*slot->finished) = 0;
 398                         if (!start_active_slot(slot)) {
 399                                 cdata->got_alternates = -1;
 400                                 slot->in_use = 0;
 401                                 if (slot->finished != NULL)
 402                                         (*slot->finished) = 1;
 403                         }
 404                         return;
 405                 }
 406         } else if (slot->curl_result != CURLE_OK) {
 407                 if (!missing_target(slot)) {
 408                         cdata->got_alternates = -1;
 409                         return;
 410                 }
 411         }
 412
 413         fwrite_buffer(&null_byte, 1, 1, alt_req->buffer);
 414         alt_req->buffer->len--;
 415         data = alt_req->buffer->buf;
 416
 417         while (i < alt_req->buffer->len) {
 418                 int posn = i;
 419                 while (posn < alt_req->buffer->len && data[posn] != '\n')
 420                         posn++;
 421                 if (data[posn] == '\n') {
 422                         int okay = 0;
 423                         int serverlen = 0;
 424                         struct alt_base *newalt;
 425                         char *target = NULL;
 426                         if (data[i] == '/') {
 427                                 /*
 428                                  * This counts
 429                                  * http://git.host/pub/scm/linux.git/
 430                                  * -----------here^
 431                                  * so memcpy(dst, base, serverlen) will
 432                                  * copy up to "...git.host".
 433                                  */
 434                                 const char *colon_ss = strstr(base,"://");
 435                                 if (colon_ss) {
 436                                         serverlen = (strchr(colon_ss + 3, '/')
 437                                                      - base);
 438                                         okay = 1;
 439                                 }
 440                         } else if (!memcmp(data + i, "../", 3)) {
 441                                 /*
 442                                  * Relative URL; chop the corresponding
 443                                  * number of subpath from base (and ../
 444                                  * from data), and concatenate the result.
 445                                  *
 446                                  * The code first drops ../ from data, and
 447                                  * then drops one ../ from data and one path
 448                                  * from base.  IOW, one extra ../ is dropped
 449                                  * from data than path is dropped from base.
 450                                  *
 451                                  * This is not wrong.  The alternate in
 452                                  *     http://git.host/pub/scm/linux.git/
 453                                  * to borrow from
 454                                  *     http://git.host/pub/scm/linus.git/
 455                                  * is ../../linus.git/objects/.  You need
 456                                  * two ../../ to borrow from your direct
 457                                  * neighbour.
 458                                  */
 459                                 i += 3;
 460                                 serverlen = strlen(base);
 461                                 while (i + 2 < posn &&
 462                                        !memcmp(data + i, "../", 3)) {
 463                                         do {
 464                                                 serverlen--;
 465                                         } while (serverlen &&
 466                                                  base[serverlen - 1] != '/');
 467                                         i += 3;
 468                                 }
 469                                 /* If the server got removed, give up. */
 470                                 okay = strchr(base, ':') - base + 3 <
 471                                        serverlen;
 472                         } else if (alt_req->http_specific) {
 473                                 char *colon = strchr(data + i, ':');
 474                                 char *slash = strchr(data + i, '/');
 475                                 if (colon && slash && colon < data + posn &&
 476                                     slash < data + posn && colon < slash) {
 477                                         okay = 1;
 478                                 }
 479                         }
 480                         /* skip "objects\n" at end */
 481                         if (okay) {
 482                                 target = xmalloc(serverlen + posn - i - 6);
 483                                 memcpy(target, base, serverlen);
 484                                 memcpy(target + serverlen, data + i,
 485                                        posn - i - 7);
 486                                 target[serverlen + posn - i - 7] = 0;
 487                                 if (walker->get_verbosely)
 488                                         fprintf(stderr,
 489                                                 "Also look at %s\n", target);
 490                                 newalt = xmalloc(sizeof(*newalt));
 491                                 newalt->next = NULL;
 492                                 newalt->base = target;
 493                                 newalt->got_indices = 0;
 494                                 newalt->packs = NULL;
 495
 496                                 while (tail->next != NULL)
 497                                         tail = tail->next;
 498                                 tail->next = newalt;
 499                         }
 500                 }
 501                 i = posn + 1;
 502         }
 503
 504         cdata->got_alternates = 1;
 505 }
 506
 507 static void fetch_alternates(struct walker *walker, const char *base)
 508 {
 509         struct strbuf buffer = STRBUF_INIT;
 510         char *url;
 511         struct active_request_slot *slot;
 512         struct alternates_request alt_req;
 513         struct walker_data *cdata = walker->data;
 514
 515         /*
 516          * If another request has already started fetching alternates,
 517          * wait for them to arrive and return to processing this request's
 518          * curl message
 519          */
 520 #ifdef USE_CURL_MULTI
 521         while (cdata->got_alternates == 0) {
 522                 step_active_slots();
 523         }
 524 #endif
 525
 526         /* Nothing to do if they've already been fetched */
 527         if (cdata->got_alternates == 1)
 528                 return;
 529
 530         /* Start the fetch */
 531         cdata->got_alternates = 0;
 532
 533         if (walker->get_verbosely)
 534                 fprintf(stderr, "Getting alternates list for %s\n", base);
 535
 536         url = xmalloc(strlen(base) + 31);
 537         sprintf(url, "%s/objects/info/http-alternates", base);
 538
 539         /*
 540          * Use a callback to process the result, since another request
 541          * may fail and need to have alternates loaded before continuing
 542          */
 543         slot = get_active_slot();
 544         slot->callback_func = process_alternates_response;
 545         alt_req.walker = walker;
 546         slot->callback_data = &alt_req;
 547
 548         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
 549         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
 550         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 551
 552         alt_req.base = base;
 553         alt_req.url = url;
 554         alt_req.buffer = &buffer;
 555         alt_req.http_specific = 1;
 556         alt_req.slot = slot;
 557
 558         if (start_active_slot(slot))
 559                 run_active_slot(slot);
 560         else
 561                 cdata->got_alternates = -1;
 562
 563         strbuf_release(&buffer);
 564         free(url);
 565 }
 566
 567 static int fetch_indices(struct walker *walker, struct alt_base *repo)
 568 {
 569         int ret;
 570
 571         if (repo->got_indices)
 572                 return 0;
 573
 574         if (walker->get_verbosely)
 575                 fprintf(stderr, "Getting pack list for %s\n", repo->base);
 576
 577         switch (http_get_info_packs(repo->base, &repo->packs)) {
 578         case HTTP_OK:
 579         case HTTP_MISSING_TARGET:
 580                 repo->got_indices = 1;
 581                 ret = 0;
 582                 break;
 583         default:
 584                 repo->got_indices = 0;
 585                 ret = -1;
 586         }
 587
 588         return ret;
 589 }
 590
 591 static int fetch_pack(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 592 {
 593         char *url;
 594         struct packed_git *target;
 595         struct packed_git **lst;
 596         FILE *packfile;
 597         char *filename;
 598         char tmpfile[PATH_MAX];
 599         int ret;
 600         long prev_posn = 0;
 601         char range[RANGE_HEADER_SIZE];
 602         struct curl_slist *range_header = NULL;
 603
 604         struct active_request_slot *slot;
 605         struct slot_results results;
 606
 607         if (fetch_indices(walker, repo))
 608                 return -1;
 609         target = find_sha1_pack(sha1, repo->packs);
 610         if (!target)
 611                 return -1;
 612
 613         if (walker->get_verbosely) {
 614                 fprintf(stderr, "Getting pack %s\n",
 615                         sha1_to_hex(target->sha1));
 616                 fprintf(stderr, " which contains %s\n",
 617                         sha1_to_hex(sha1));
 618         }
 619
 620         url = xmalloc(strlen(repo->base) + 65);
 621         sprintf(url, "%s/objects/pack/pack-%s.pack",
 622                 repo->base, sha1_to_hex(target->sha1));
 623
 624         filename = sha1_pack_name(target->sha1);
 625         snprintf(tmpfile, sizeof(tmpfile), "%s.temp", filename);
 626         packfile = fopen(tmpfile, "a");
 627         if (!packfile)
 628                 return error("Unable to open local file %s for pack",
 629                              tmpfile);
 630
 631         slot = get_active_slot();
 632         slot->results = &results;
 633         curl_easy_setopt(slot->curl, CURLOPT_FILE, packfile);
 634         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
 635         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
 636         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
 637         slot->local = packfile;
 638
 639         /*
 640          * If there is data present from a previous transfer attempt,
 641          * resume where it left off
 642          */
 643         prev_posn = ftell(packfile);
 644         if (prev_posn>0) {
 645                 if (walker->get_verbosely)
 646                         fprintf(stderr,
 647                                 "Resuming fetch of pack %s at byte %ld\n",
 648                                 sha1_to_hex(target->sha1), prev_posn);
 649                 sprintf(range, "Range: bytes=%ld-", prev_posn);
 650                 range_header = curl_slist_append(range_header, range);
 651                 curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, range_header);
 652         }
 653
 654         if (start_active_slot(slot)) {
 655                 run_active_slot(slot);
 656                 if (results.curl_result != CURLE_OK) {
 657                         fclose(packfile);
 658                         slot->local = NULL;
 659                         return error("Unable to get pack file %s\n%s", url,
 660                                      curl_errorstr);
 661                 }
 662         } else {
 663                 fclose(packfile);
 664                 slot->local = NULL;
 665                 return error("Unable to start request");
 666         }
 667
 668         target->pack_size = ftell(packfile);
 669         fclose(packfile);
 670         slot->local = NULL;
 671
 672         ret = move_temp_to_file(tmpfile, filename);
 673         if (ret)
 674                 return ret;
 675
 676         lst = &repo->packs;
 677         while (*lst != target)
 678                 lst = &((*lst)->next);
 679         *lst = (*lst)->next;
 680
 681         if (verify_pack(target))
 682                 return -1;
 683         install_packed_git(target);
 684
 685         return 0;
 686 }
 687
 688 static void abort_object_request(struct object_request *obj_req)
 689 {
 690         if (obj_req->local >= 0) {
 691                 close(obj_req->local);
 692                 obj_req->local = -1;
 693         }
 694         unlink_or_warn(obj_req->tmpfile);
 695         if (obj_req->slot) {
 696                 release_active_slot(obj_req->slot);
 697                 obj_req->slot = NULL;
 698         }
 699         release_object_request(obj_req);
 700 }
 701
 702 static int fetch_object(struct walker *walker, struct alt_base *repo, unsigned char *sha1)
 703 {
 704         char *hex = sha1_to_hex(sha1);
 705         int ret = 0;
 706         struct object_request *obj_req = object_queue_head;
 707
 708         while (obj_req != NULL && hashcmp(obj_req->sha1, sha1))
 709                 obj_req = obj_req->next;
 710         if (obj_req == NULL)
 711                 return error("Couldn't find request for %s in the queue", hex);
 712
 713         if (has_sha1_file(obj_req->sha1)) {
 714                 abort_object_request(obj_req);
 715                 return 0;
 716         }
 717
 718 #ifdef USE_CURL_MULTI
 719         while (obj_req->state == WAITING)
 720                 step_active_slots();
 721 #else
 722         start_object_request(walker, obj_req);
 723 #endif
 724
 725         while (obj_req->state == ACTIVE)
 726                 run_active_slot(obj_req->slot);
 727
 728         if (obj_req->local != -1) {
 729                 close(obj_req->local);
 730                 obj_req->local = -1;
 731         }
 732
 733         if (obj_req->state == ABORTED) {
 734                 ret = error("Request for %s aborted", hex);
 735         } else if (obj_req->curl_result != CURLE_OK &&
 736                    obj_req->http_code != 416) {
 737                 if (missing_target(obj_req))
 738                         ret = -1; /* Be silent, it is probably in a pack. */
 739                 else
 740                         ret = error("%s (curl_result = %d, http_code = %ld, sha1 = %s)",
 741                                     obj_req->errorstr, obj_req->curl_result,
 742                                     obj_req->http_code, hex);
 743         } else if (obj_req->zret != Z_STREAM_END) {
 744                 walker->corrupt_object_found++;
 745                 ret = error("File %s (%s) corrupt", hex, obj_req->url);
 746         } else if (hashcmp(obj_req->sha1, obj_req->real_sha1)) {
 747                 ret = error("File %s has bad hash", hex);
 748         } else if (obj_req->rename < 0) {
 749                 ret = error("unable to write sha1 filename %s",
 750                             obj_req->filename);
 751         }
 752
 753         release_object_request(obj_req);
 754         return ret;
 755 }
 756
 757 static int fetch(struct walker *walker, unsigned char *sha1)
 758 {
 759         struct walker_data *data = walker->data;
 760         struct alt_base *altbase = data->alt;
 761
 762         if (!fetch_object(walker, altbase, sha1))
 763                 return 0;
 764         while (altbase) {
 765                 if (!fetch_pack(walker, altbase, sha1))
 766                         return 0;
 767                 fetch_alternates(walker, data->alt->base);
 768                 altbase = altbase->next;
 769         }
 770         return error("Unable to find %s under %s", sha1_to_hex(sha1),
 771                      data->alt->base);
 772 }
 773
 774 static int fetch_ref(struct walker *walker, struct ref *ref)
 775 {
 776         struct walker_data *data = walker->data;
 777         return http_fetch_ref(data->alt->base, ref);
 778 }
 779
 780 static void cleanup(struct walker *walker)
 781 {
 782         http_cleanup();
 783 }
 784
 785 struct walker *get_http_walker(const char *url, struct remote *remote)
 786 {
 787         char *s;
 788         struct walker_data *data = xmalloc(sizeof(struct walker_data));
 789         struct walker *walker = xmalloc(sizeof(struct walker));
 790
 791         http_init(remote);
 792
 793         data->alt = xmalloc(sizeof(*data->alt));
 794         data->alt->base = xmalloc(strlen(url) + 1);
 795         strcpy(data->alt->base, url);
 796         for (s = data->alt->base + strlen(data->alt->base) - 1; *s == '/'; --s)
 797                 *s = 0;
 798
 799         data->alt->got_indices = 0;
 800         data->alt->packs = NULL;
 801         data->alt->next = NULL;
 802         data->got_alternates = -1;
 803
 804         walker->corrupt_object_found = 0;
 805         walker->fetch = fetch;
 806         walker->fetch_ref = fetch_ref;
 807         walker->prefetch = prefetch;
 808         walker->cleanup = cleanup;
 809         walker->data = data;
 810
 811 #ifdef USE_CURL_MULTI
 812         add_fill_function(walker, (int (*)(void *)) fill_active_slot);
 813 #endif
 814
 815         return walker;
 816 }