builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13
  14 static int transfer_unpack_limit = -1;
  15 static int fetch_unpack_limit = -1;
  16 static int unpack_limit = 100;
  17 static int prefer_ofs_delta = 1;
  18 static int no_done = 0;
  19 static struct fetch_pack_args args = {
  20         /* .uploadpack = */ "git-upload-pack",
  21 };
  22
  23 static const char fetch_pack_usage[] =
  24 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  25
  26 #define COMPLETE        (1U << 0)
  27 #define COMMON          (1U << 1)
  28 #define COMMON_REF      (1U << 2)
  29 #define SEEN            (1U << 3)
  30 #define POPPED          (1U << 4)
  31
  32 static int marked;
  33
  34 /*
  35  * After sending this many "have"s if we do not get any new ACK , we
  36  * give up traversing our history.
  37  */
  38 #define MAX_IN_VAIN 256
  39
  40 static struct commit_list *rev_list;
  41 static int non_common_revs, multi_ack, use_sideband;
  42
  43 static void rev_list_push(struct commit *commit, int mark)
  44 {
  45         if (!(commit->object.flags & mark)) {
  46                 commit->object.flags |= mark;
  47
  48                 if (!(commit->object.parsed))
  49                         if (parse_commit(commit))
  50                                 return;
  51
  52                 commit_list_insert_by_date(commit, &rev_list);
  53
  54                 if (!(commit->object.flags & COMMON))
  55                         non_common_revs++;
  56         }
  57 }
  58
  59 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  60 {
  61         struct object *o = deref_tag(parse_object(sha1), path, 0);
  62
  63         if (o && o->type == OBJ_COMMIT)
  64                 rev_list_push((struct commit *)o, SEEN);
  65
  66         return 0;
  67 }
  68
  69 static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  70 {
  71         struct object *o = deref_tag(parse_object(sha1), path, 0);
  72
  73         if (o && o->type == OBJ_COMMIT)
  74                 clear_commit_marks((struct commit *)o,
  75                                    COMMON | COMMON_REF | SEEN | POPPED);
  76         return 0;
  77 }
  78
  79 /*
  80    This function marks a rev and its ancestors as common.
  81    In some cases, it is desirable to mark only the ancestors (for example
  82    when only the server does not yet know that they are common).
  83 */
  84
  85 static void mark_common(struct commit *commit,
  86                 int ancestors_only, int dont_parse)
  87 {
  88         if (commit != NULL && !(commit->object.flags & COMMON)) {
  89                 struct object *o = (struct object *)commit;
  90
  91                 if (!ancestors_only)
  92                         o->flags |= COMMON;
  93
  94                 if (!(o->flags & SEEN))
  95                         rev_list_push(commit, SEEN);
  96                 else {
  97                         struct commit_list *parents;
  98
  99                         if (!ancestors_only && !(o->flags & POPPED))
 100                                 non_common_revs--;
 101                         if (!o->parsed && !dont_parse)
 102                                 if (parse_commit(commit))
 103                                         return;
 104
 105                         for (parents = commit->parents;
 106                                         parents;
 107                                         parents = parents->next)
 108                                 mark_common(parents->item, 0, dont_parse);
 109                 }
 110         }
 111 }
 112
 113 /*
 114   Get the next rev to send, ignoring the common.
 115 */
 116
 117 static const unsigned char *get_rev(void)
 118 {
 119         struct commit *commit = NULL;
 120
 121         while (commit == NULL) {
 122                 unsigned int mark;
 123                 struct commit_list *parents;
 124
 125                 if (rev_list == NULL || non_common_revs == 0)
 126                         return NULL;
 127
 128                 commit = rev_list->item;
 129                 if (!commit->object.parsed)
 130                         parse_commit(commit);
 131                 parents = commit->parents;
 132
 133                 commit->object.flags |= POPPED;
 134                 if (!(commit->object.flags & COMMON))
 135                         non_common_revs--;
 136
 137                 if (commit->object.flags & COMMON) {
 138                         /* do not send "have", and ignore ancestors */
 139                         commit = NULL;
 140                         mark = COMMON | SEEN;
 141                 } else if (commit->object.flags & COMMON_REF)
 142                         /* send "have", and ignore ancestors */
 143                         mark = COMMON | SEEN;
 144                 else
 145                         /* send "have", also for its ancestors */
 146                         mark = SEEN;
 147
 148                 while (parents) {
 149                         if (!(parents->item->object.flags & SEEN))
 150                                 rev_list_push(parents->item, mark);
 151                         if (mark & COMMON)
 152                                 mark_common(parents->item, 1, 0);
 153                         parents = parents->next;
 154                 }
 155
 156                 rev_list = rev_list->next;
 157         }
 158
 159         return commit->object.sha1;
 160 }
 161
 162 enum ack_type {
 163         NAK = 0,
 164         ACK,
 165         ACK_continue,
 166         ACK_common,
 167         ACK_ready
 168 };
 169
 170 static void consume_shallow_list(int fd)
 171 {
 172         if (args.stateless_rpc && args.depth > 0) {
 173                 /* If we sent a depth we will get back "duplicate"
 174                  * shallow and unshallow commands every time there
 175                  * is a block of have lines exchanged.
 176                  */
 177                 char line[1000];
 178                 while (packet_read_line(fd, line, sizeof(line))) {
 179                         if (!prefixcmp(line, "shallow "))
 180                                 continue;
 181                         if (!prefixcmp(line, "unshallow "))
 182                                 continue;
 183                         die("git fetch-pack: expected shallow list");
 184                 }
 185         }
 186 }
 187
 188 struct write_shallow_data {
 189         struct strbuf *out;
 190         int use_pack_protocol;
 191         int count;
 192 };
 193
 194 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 195 {
 196         struct write_shallow_data *data = cb_data;
 197         const char *hex = sha1_to_hex(graft->sha1);
 198         data->count++;
 199         if (data->use_pack_protocol)
 200                 packet_buf_write(data->out, "shallow %s", hex);
 201         else {
 202                 strbuf_addstr(data->out, hex);
 203                 strbuf_addch(data->out, '\n');
 204         }
 205         return 0;
 206 }
 207
 208 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 209 {
 210         struct write_shallow_data data;
 211         data.out = out;
 212         data.use_pack_protocol = use_pack_protocol;
 213         data.count = 0;
 214         for_each_commit_graft(write_one_shallow, &data);
 215         return data.count;
 216 }
 217
 218 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 219 {
 220         static char line[1000];
 221         int len = packet_read_line(fd, line, sizeof(line));
 222
 223         if (!len)
 224                 die("git fetch-pack: expected ACK/NAK, got EOF");
 225         if (line[len-1] == '\n')
 226                 line[--len] = 0;
 227         if (!strcmp(line, "NAK"))
 228                 return NAK;
 229         if (!prefixcmp(line, "ACK ")) {
 230                 if (!get_sha1_hex(line+4, result_sha1)) {
 231                         if (strstr(line+45, "continue"))
 232                                 return ACK_continue;
 233                         if (strstr(line+45, "common"))
 234                                 return ACK_common;
 235                         if (strstr(line+45, "ready"))
 236                                 return ACK_ready;
 237                         return ACK;
 238                 }
 239         }
 240         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 241 }
 242
 243 static void send_request(int fd, struct strbuf *buf)
 244 {
 245         if (args.stateless_rpc) {
 246                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 247                 packet_flush(fd);
 248         } else
 249                 safe_write(fd, buf->buf, buf->len);
 250 }
 251
 252 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 253 {
 254         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 255 }
 256
 257 static void insert_alternate_refs(void)
 258 {
 259         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 260 }
 261
 262 #define INITIAL_FLUSH 16
 263 #define PIPESAFE_FLUSH 32
 264 #define LARGE_FLUSH 1024
 265
 266 static int next_flush(int count)
 267 {
 268         int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 269
 270         if (count < flush_limit)
 271                 count <<= 1;
 272         else
 273                 count += flush_limit;
 274         return count;
 275 }
 276
 277 static int find_common(int fd[2], unsigned char *result_sha1,
 278                        struct ref *refs)
 279 {
 280         int fetching;
 281         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 282         const unsigned char *sha1;
 283         unsigned in_vain = 0;
 284         int got_continue = 0;
 285         int got_ready = 0;
 286         struct strbuf req_buf = STRBUF_INIT;
 287         size_t state_len = 0;
 288
 289         if (args.stateless_rpc && multi_ack == 1)
 290                 die("--stateless-rpc requires multi_ack_detailed");
 291         if (marked)
 292                 for_each_ref(clear_marks, NULL);
 293         marked = 1;
 294
 295         for_each_ref(rev_list_insert_ref, NULL);
 296         insert_alternate_refs();
 297
 298         fetching = 0;
 299         for ( ; refs ; refs = refs->next) {
 300                 unsigned char *remote = refs->old_sha1;
 301                 const char *remote_hex;
 302                 struct object *o;
 303
 304                 /*
 305                  * If that object is complete (i.e. it is an ancestor of a
 306                  * local ref), we tell them we have it but do not have to
 307                  * tell them about its ancestors, which they already know
 308                  * about.
 309                  *
 310                  * We use lookup_object here because we are only
 311                  * interested in the case we *know* the object is
 312                  * reachable and we have already scanned it.
 313                  */
 314                 if (((o = lookup_object(remote)) != NULL) &&
 315                                 (o->flags & COMPLETE)) {
 316                         continue;
 317                 }
 318
 319                 remote_hex = sha1_to_hex(remote);
 320                 if (!fetching) {
 321                         struct strbuf c = STRBUF_INIT;
 322                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 323                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 324                         if (no_done)            strbuf_addstr(&c, " no-done");
 325                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 326                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 327                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 328                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 329                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 330                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 331                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 332                         strbuf_release(&c);
 333                 } else
 334                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 335                 fetching++;
 336         }
 337
 338         if (!fetching) {
 339                 strbuf_release(&req_buf);
 340                 packet_flush(fd[1]);
 341                 return 1;
 342         }
 343
 344         if (is_repository_shallow())
 345                 write_shallow_commits(&req_buf, 1);
 346         if (args.depth > 0)
 347                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 348         packet_buf_flush(&req_buf);
 349         state_len = req_buf.len;
 350
 351         if (args.depth > 0) {
 352                 char line[1024];
 353                 unsigned char sha1[20];
 354
 355                 send_request(fd[1], &req_buf);
 356                 while (packet_read_line(fd[0], line, sizeof(line))) {
 357                         if (!prefixcmp(line, "shallow ")) {
 358                                 if (get_sha1_hex(line + 8, sha1))
 359                                         die("invalid shallow line: %s", line);
 360                                 register_shallow(sha1);
 361                                 continue;
 362                         }
 363                         if (!prefixcmp(line, "unshallow ")) {
 364                                 if (get_sha1_hex(line + 10, sha1))
 365                                         die("invalid unshallow line: %s", line);
 366                                 if (!lookup_object(sha1))
 367                                         die("object not found: %s", line);
 368                                 /* make sure that it is parsed as shallow */
 369                                 if (!parse_object(sha1))
 370                                         die("error in object: %s", line);
 371                                 if (unregister_shallow(sha1))
 372                                         die("no shallow found: %s", line);
 373                                 continue;
 374                         }
 375                         die("expected shallow/unshallow, got %s", line);
 376                 }
 377         } else if (!args.stateless_rpc)
 378                 send_request(fd[1], &req_buf);
 379
 380         if (!args.stateless_rpc) {
 381                 /* If we aren't using the stateless-rpc interface
 382                  * we don't need to retain the headers.
 383                  */
 384                 strbuf_setlen(&req_buf, 0);
 385                 state_len = 0;
 386         }
 387
 388         flushes = 0;
 389         retval = -1;
 390         while ((sha1 = get_rev())) {
 391                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 392                 if (args.verbose)
 393                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 394                 in_vain++;
 395                 if (flush_at <= ++count) {
 396                         int ack;
 397
 398                         packet_buf_flush(&req_buf);
 399                         send_request(fd[1], &req_buf);
 400                         strbuf_setlen(&req_buf, state_len);
 401                         flushes++;
 402                         flush_at = next_flush(count);
 403
 404                         /*
 405                          * We keep one window "ahead" of the other side, and
 406                          * will wait for an ACK only on the next one
 407                          */
 408                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 409                                 continue;
 410
 411                         consume_shallow_list(fd[0]);
 412                         do {
 413                                 ack = get_ack(fd[0], result_sha1);
 414                                 if (args.verbose && ack)
 415                                         fprintf(stderr, "got ack %d %s\n", ack,
 416                                                         sha1_to_hex(result_sha1));
 417                                 switch (ack) {
 418                                 case ACK:
 419                                         flushes = 0;
 420                                         multi_ack = 0;
 421                                         retval = 0;
 422                                         goto done;
 423                                 case ACK_common:
 424                                 case ACK_ready:
 425                                 case ACK_continue: {
 426                                         struct commit *commit =
 427                                                 lookup_commit(result_sha1);
 428                                         if (!commit)
 429                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 430                                         if (args.stateless_rpc
 431                                          && ack == ACK_common
 432                                          && !(commit->object.flags & COMMON)) {
 433                                                 /* We need to replay the have for this object
 434                                                  * on the next RPC request so the peer knows
 435                                                  * it is in common with us.
 436                                                  */
 437                                                 const char *hex = sha1_to_hex(result_sha1);
 438                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 439                                                 state_len = req_buf.len;
 440                                         }
 441                                         mark_common(commit, 0, 1);
 442                                         retval = 0;
 443                                         in_vain = 0;
 444                                         got_continue = 1;
 445                                         if (ack == ACK_ready) {
 446                                                 rev_list = NULL;
 447                                                 got_ready = 1;
 448                                         }
 449                                         break;
 450                                         }
 451                                 }
 452                         } while (ack);
 453                         flushes--;
 454                         if (got_continue && MAX_IN_VAIN < in_vain) {
 455                                 if (args.verbose)
 456                                         fprintf(stderr, "giving up\n");
 457                                 break; /* give up */
 458                         }
 459                 }
 460         }
 461 done:
 462         if (!got_ready || !no_done) {
 463                 packet_buf_write(&req_buf, "done\n");
 464                 send_request(fd[1], &req_buf);
 465         }
 466         if (args.verbose)
 467                 fprintf(stderr, "done\n");
 468         if (retval != 0) {
 469                 multi_ack = 0;
 470                 flushes++;
 471         }
 472         strbuf_release(&req_buf);
 473
 474         consume_shallow_list(fd[0]);
 475         while (flushes || multi_ack) {
 476                 int ack = get_ack(fd[0], result_sha1);
 477                 if (ack) {
 478                         if (args.verbose)
 479                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 480                                         sha1_to_hex(result_sha1));
 481                         if (ack == ACK)
 482                                 return 0;
 483                         multi_ack = 1;
 484                         continue;
 485                 }
 486                 flushes--;
 487         }
 488         /* it is no error to fetch into a completely empty repo */
 489         return count ? retval : 0;
 490 }
 491
 492 static struct commit_list *complete;
 493
 494 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 495 {
 496         struct object *o = parse_object(sha1);
 497
 498         while (o && o->type == OBJ_TAG) {
 499                 struct tag *t = (struct tag *) o;
 500                 if (!t->tagged)
 501                         break; /* broken repository */
 502                 o->flags |= COMPLETE;
 503                 o = parse_object(t->tagged->sha1);
 504         }
 505         if (o && o->type == OBJ_COMMIT) {
 506                 struct commit *commit = (struct commit *)o;
 507                 if (!(commit->object.flags & COMPLETE)) {
 508                         commit->object.flags |= COMPLETE;
 509                         commit_list_insert_by_date(commit, &complete);
 510                 }
 511         }
 512         return 0;
 513 }
 514
 515 static void mark_recent_complete_commits(unsigned long cutoff)
 516 {
 517         while (complete && cutoff <= complete->item->date) {
 518                 if (args.verbose)
 519                         fprintf(stderr, "Marking %s as complete\n",
 520                                 sha1_to_hex(complete->item->object.sha1));
 521                 pop_most_recent_commit(&complete, COMPLETE);
 522         }
 523 }
 524
 525 static void filter_refs(struct ref **refs, int nr_match, char **match)
 526 {
 527         struct ref **return_refs;
 528         struct ref *newlist = NULL;
 529         struct ref **newtail = &newlist;
 530         struct ref *ref, *next;
 531         struct ref *fastarray[32];
 532
 533         if (nr_match && !args.fetch_all) {
 534                 if (ARRAY_SIZE(fastarray) < nr_match)
 535                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 536                 else {
 537                         return_refs = fastarray;
 538                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 539                 }
 540         }
 541         else
 542                 return_refs = NULL;
 543
 544         for (ref = *refs; ref; ref = next) {
 545                 next = ref->next;
 546                 if (!memcmp(ref->name, "refs/", 5) &&
 547                     check_ref_format(ref->name + 5))
 548                         ; /* trash */
 549                 else if (args.fetch_all &&
 550                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 551                         *newtail = ref;
 552                         ref->next = NULL;
 553                         newtail = &ref->next;
 554                         continue;
 555                 }
 556                 else {
 557                         int order = path_match(ref->name, nr_match, match);
 558                         if (order) {
 559                                 return_refs[order-1] = ref;
 560                                 continue; /* we will link it later */
 561                         }
 562                 }
 563                 free(ref);
 564         }
 565
 566         if (!args.fetch_all) {
 567                 int i;
 568                 for (i = 0; i < nr_match; i++) {
 569                         ref = return_refs[i];
 570                         if (ref) {
 571                                 *newtail = ref;
 572                                 ref->next = NULL;
 573                                 newtail = &ref->next;
 574                         }
 575                 }
 576                 if (return_refs != fastarray)
 577                         free(return_refs);
 578         }
 579         *refs = newlist;
 580 }
 581
 582 static int everything_local(struct ref **refs, int nr_match, char **match)
 583 {
 584         struct ref *ref;
 585         int retval;
 586         unsigned long cutoff = 0;
 587
 588         save_commit_buffer = 0;
 589
 590         for (ref = *refs; ref; ref = ref->next) {
 591                 struct object *o;
 592
 593                 o = parse_object(ref->old_sha1);
 594                 if (!o)
 595                         continue;
 596
 597                 /* We already have it -- which may mean that we were
 598                  * in sync with the other side at some time after
 599                  * that (it is OK if we guess wrong here).
 600                  */
 601                 if (o->type == OBJ_COMMIT) {
 602                         struct commit *commit = (struct commit *)o;
 603                         if (!cutoff || cutoff < commit->date)
 604                                 cutoff = commit->date;
 605                 }
 606         }
 607
 608         if (!args.depth) {
 609                 for_each_ref(mark_complete, NULL);
 610                 if (cutoff)
 611                         mark_recent_complete_commits(cutoff);
 612         }
 613
 614         /*
 615          * Mark all complete remote refs as common refs.
 616          * Don't mark them common yet; the server has to be told so first.
 617          */
 618         for (ref = *refs; ref; ref = ref->next) {
 619                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 620                                              NULL, 0);
 621
 622                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 623                         continue;
 624
 625                 if (!(o->flags & SEEN)) {
 626                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 627
 628                         mark_common((struct commit *)o, 1, 1);
 629                 }
 630         }
 631
 632         filter_refs(refs, nr_match, match);
 633
 634         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 635                 const unsigned char *remote = ref->old_sha1;
 636                 unsigned char local[20];
 637                 struct object *o;
 638
 639                 o = lookup_object(remote);
 640                 if (!o || !(o->flags & COMPLETE)) {
 641                         retval = 0;
 642                         if (!args.verbose)
 643                                 continue;
 644                         fprintf(stderr,
 645                                 "want %s (%s)\n", sha1_to_hex(remote),
 646                                 ref->name);
 647                         continue;
 648                 }
 649
 650                 hashcpy(ref->new_sha1, local);
 651                 if (!args.verbose)
 652                         continue;
 653                 fprintf(stderr,
 654                         "already have %s (%s)\n", sha1_to_hex(remote),
 655                         ref->name);
 656         }
 657         return retval;
 658 }
 659
 660 static int sideband_demux(int in, int out, void *data)
 661 {
 662         int *xd = data;
 663
 664         int ret = recv_sideband("fetch-pack", xd[0], out);
 665         close(out);
 666         return ret;
 667 }
 668
 669 static int get_pack(int xd[2], char **pack_lockfile)
 670 {
 671         struct async demux;
 672         const char *argv[20];
 673         char keep_arg[256];
 674         char hdr_arg[256];
 675         const char **av;
 676         int do_keep = args.keep_pack;
 677         struct child_process cmd;
 678
 679         memset(&demux, 0, sizeof(demux));
 680         if (use_sideband) {
 681                 /* xd[] is talking with upload-pack; subprocess reads from
 682                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 683                  * through demux->out.
 684                  */
 685                 demux.proc = sideband_demux;
 686                 demux.data = xd;
 687                 demux.out = -1;
 688                 if (start_async(&demux))
 689                         die("fetch-pack: unable to fork off sideband"
 690                             " demultiplexer");
 691         }
 692         else
 693                 demux.out = xd[0];
 694
 695         memset(&cmd, 0, sizeof(cmd));
 696         cmd.argv = argv;
 697         av = argv;
 698         *hdr_arg = 0;
 699         if (!args.keep_pack && unpack_limit) {
 700                 struct pack_header header;
 701
 702                 if (read_pack_header(demux.out, &header))
 703                         die("protocol error: bad pack header");
 704                 snprintf(hdr_arg, sizeof(hdr_arg),
 705                          "--pack_header=%"PRIu32",%"PRIu32,
 706                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 707                 if (ntohl(header.hdr_entries) < unpack_limit)
 708                         do_keep = 0;
 709                 else
 710                         do_keep = 1;
 711         }
 712
 713         if (do_keep) {
 714                 if (pack_lockfile)
 715                         cmd.out = -1;
 716                 *av++ = "index-pack";
 717                 *av++ = "--stdin";
 718                 if (!args.quiet && !args.no_progress)
 719                         *av++ = "-v";
 720                 if (args.use_thin_pack)
 721                         *av++ = "--fix-thin";
 722                 if (args.lock_pack || unpack_limit) {
 723                         int s = sprintf(keep_arg,
 724                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 725                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 726                                 strcpy(keep_arg + s, "localhost");
 727                         *av++ = keep_arg;
 728                 }
 729         }
 730         else {
 731                 *av++ = "unpack-objects";
 732                 if (args.quiet)
 733                         *av++ = "-q";
 734         }
 735         if (*hdr_arg)
 736                 *av++ = hdr_arg;
 737         *av++ = NULL;
 738
 739         cmd.in = demux.out;
 740         cmd.git_cmd = 1;
 741         if (start_command(&cmd))
 742                 die("fetch-pack: unable to fork off %s", argv[0]);
 743         if (do_keep && pack_lockfile) {
 744                 *pack_lockfile = index_pack_lockfile(cmd.out);
 745                 close(cmd.out);
 746         }
 747
 748         if (finish_command(&cmd))
 749                 die("%s failed", argv[0]);
 750         if (use_sideband && finish_async(&demux))
 751                 die("error in sideband demultiplexer");
 752         return 0;
 753 }
 754
 755 static struct ref *do_fetch_pack(int fd[2],
 756                 const struct ref *orig_ref,
 757                 int nr_match,
 758                 char **match,
 759                 char **pack_lockfile)
 760 {
 761         struct ref *ref = copy_ref_list(orig_ref);
 762         unsigned char sha1[20];
 763
 764         if (is_repository_shallow() && !server_supports("shallow"))
 765                 die("Server does not support shallow clients");
 766         if (server_supports("multi_ack_detailed")) {
 767                 if (args.verbose)
 768                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 769                 multi_ack = 2;
 770                 if (server_supports("no-done")) {
 771                         if (args.verbose)
 772                                 fprintf(stderr, "Server supports no-done\n");
 773                         if (args.stateless_rpc)
 774                                 no_done = 1;
 775                 }
 776         }
 777         else if (server_supports("multi_ack")) {
 778                 if (args.verbose)
 779                         fprintf(stderr, "Server supports multi_ack\n");
 780                 multi_ack = 1;
 781         }
 782         if (server_supports("side-band-64k")) {
 783                 if (args.verbose)
 784                         fprintf(stderr, "Server supports side-band-64k\n");
 785                 use_sideband = 2;
 786         }
 787         else if (server_supports("side-band")) {
 788                 if (args.verbose)
 789                         fprintf(stderr, "Server supports side-band\n");
 790                 use_sideband = 1;
 791         }
 792         if (server_supports("ofs-delta")) {
 793                 if (args.verbose)
 794                         fprintf(stderr, "Server supports ofs-delta\n");
 795         } else
 796                 prefer_ofs_delta = 0;
 797         if (everything_local(&ref, nr_match, match)) {
 798                 packet_flush(fd[1]);
 799                 goto all_done;
 800         }
 801         if (find_common(fd, sha1, ref) < 0)
 802                 if (!args.keep_pack)
 803                         /* When cloning, it is not unusual to have
 804                          * no common commit.
 805                          */
 806                         warning("no common commits");
 807
 808         if (args.stateless_rpc)
 809                 packet_flush(fd[1]);
 810         if (get_pack(fd, pack_lockfile))
 811                 die("git fetch-pack: fetch failed.");
 812
 813  all_done:
 814         return ref;
 815 }
 816
 817 static int remove_duplicates(int nr_heads, char **heads)
 818 {
 819         int src, dst;
 820
 821         for (src = dst = 0; src < nr_heads; src++) {
 822                 /* If heads[src] is different from any of
 823                  * heads[0..dst], push it in.
 824                  */
 825                 int i;
 826                 for (i = 0; i < dst; i++) {
 827                         if (!strcmp(heads[i], heads[src]))
 828                                 break;
 829                 }
 830                 if (i < dst)
 831                         continue;
 832                 if (src != dst)
 833                         heads[dst] = heads[src];
 834                 dst++;
 835         }
 836         return dst;
 837 }
 838
 839 static int fetch_pack_config(const char *var, const char *value, void *cb)
 840 {
 841         if (strcmp(var, "fetch.unpacklimit") == 0) {
 842                 fetch_unpack_limit = git_config_int(var, value);
 843                 return 0;
 844         }
 845
 846         if (strcmp(var, "transfer.unpacklimit") == 0) {
 847                 transfer_unpack_limit = git_config_int(var, value);
 848                 return 0;
 849         }
 850
 851         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 852                 prefer_ofs_delta = git_config_bool(var, value);
 853                 return 0;
 854         }
 855
 856         return git_default_config(var, value, cb);
 857 }
 858
 859 static struct lock_file lock;
 860
 861 static void fetch_pack_setup(void)
 862 {
 863         static int did_setup;
 864         if (did_setup)
 865                 return;
 866         git_config(fetch_pack_config, NULL);
 867         if (0 <= transfer_unpack_limit)
 868                 unpack_limit = transfer_unpack_limit;
 869         else if (0 <= fetch_unpack_limit)
 870                 unpack_limit = fetch_unpack_limit;
 871         did_setup = 1;
 872 }
 873
 874 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 875 {
 876         int i, ret, nr_heads;
 877         struct ref *ref = NULL;
 878         char *dest = NULL, **heads;
 879         int fd[2];
 880         char *pack_lockfile = NULL;
 881         char **pack_lockfile_ptr = NULL;
 882         struct child_process *conn;
 883
 884         packet_trace_identity("fetch-pack");
 885
 886         nr_heads = 0;
 887         heads = NULL;
 888         for (i = 1; i < argc; i++) {
 889                 const char *arg = argv[i];
 890
 891                 if (*arg == '-') {
 892                         if (!prefixcmp(arg, "--upload-pack=")) {
 893                                 args.uploadpack = arg + 14;
 894                                 continue;
 895                         }
 896                         if (!prefixcmp(arg, "--exec=")) {
 897                                 args.uploadpack = arg + 7;
 898                                 continue;
 899                         }
 900                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 901                                 args.quiet = 1;
 902                                 continue;
 903                         }
 904                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 905                                 args.lock_pack = args.keep_pack;
 906                                 args.keep_pack = 1;
 907                                 continue;
 908                         }
 909                         if (!strcmp("--thin", arg)) {
 910                                 args.use_thin_pack = 1;
 911                                 continue;
 912                         }
 913                         if (!strcmp("--include-tag", arg)) {
 914                                 args.include_tag = 1;
 915                                 continue;
 916                         }
 917                         if (!strcmp("--all", arg)) {
 918                                 args.fetch_all = 1;
 919                                 continue;
 920                         }
 921                         if (!strcmp("-v", arg)) {
 922                                 args.verbose = 1;
 923                                 continue;
 924                         }
 925                         if (!prefixcmp(arg, "--depth=")) {
 926                                 args.depth = strtol(arg + 8, NULL, 0);
 927                                 continue;
 928                         }
 929                         if (!strcmp("--no-progress", arg)) {
 930                                 args.no_progress = 1;
 931                                 continue;
 932                         }
 933                         if (!strcmp("--stateless-rpc", arg)) {
 934                                 args.stateless_rpc = 1;
 935                                 continue;
 936                         }
 937                         if (!strcmp("--lock-pack", arg)) {
 938                                 args.lock_pack = 1;
 939                                 pack_lockfile_ptr = &pack_lockfile;
 940                                 continue;
 941                         }
 942                         usage(fetch_pack_usage);
 943                 }
 944                 dest = (char *)arg;
 945                 heads = (char **)(argv + i + 1);
 946                 nr_heads = argc - i - 1;
 947                 break;
 948         }
 949         if (!dest)
 950                 usage(fetch_pack_usage);
 951
 952         if (args.stateless_rpc) {
 953                 conn = NULL;
 954                 fd[0] = 0;
 955                 fd[1] = 1;
 956         } else {
 957                 conn = git_connect(fd, (char *)dest, args.uploadpack,
 958                                    args.verbose ? CONNECT_VERBOSE : 0);
 959         }
 960
 961         get_remote_heads(fd[0], &ref, 0, NULL, 0, NULL);
 962
 963         ref = fetch_pack(&args, fd, conn, ref, dest,
 964                 nr_heads, heads, pack_lockfile_ptr);
 965         if (pack_lockfile) {
 966                 printf("lock %s\n", pack_lockfile);
 967                 fflush(stdout);
 968         }
 969         close(fd[0]);
 970         close(fd[1]);
 971         if (finish_connect(conn))
 972                 ref = NULL;
 973         ret = !ref;
 974
 975         if (!ret && nr_heads) {
 976                 /* If the heads to pull were given, we should have
 977                  * consumed all of them by matching the remote.
 978                  * Otherwise, 'git fetch remote no-such-ref' would
 979                  * silently succeed without issuing an error.
 980                  */
 981                 for (i = 0; i < nr_heads; i++)
 982                         if (heads[i] && heads[i][0]) {
 983                                 error("no such remote ref %s", heads[i]);
 984                                 ret = 1;
 985                         }
 986         }
 987         while (ref) {
 988                 printf("%s %s\n",
 989                        sha1_to_hex(ref->old_sha1), ref->name);
 990                 ref = ref->next;
 991         }
 992
 993         return ret;
 994 }
 995
 996 struct ref *fetch_pack(struct fetch_pack_args *my_args,
 997                        int fd[], struct child_process *conn,
 998                        const struct ref *ref,
 999                 const char *dest,
1000                 int nr_heads,
1001                 char **heads,
1002                 char **pack_lockfile)
1003 {
1004         struct stat st;
1005         struct ref *ref_cpy;
1006
1007         fetch_pack_setup();
1008         if (&args != my_args)
1009                 memcpy(&args, my_args, sizeof(args));
1010         if (args.depth > 0) {
1011                 if (stat(git_path("shallow"), &st))
1012                         st.st_mtime = 0;
1013         }
1014
1015         if (heads && nr_heads)
1016                 nr_heads = remove_duplicates(nr_heads, heads);
1017         if (!ref) {
1018                 packet_flush(fd[1]);
1019                 die("no matching remote head");
1020         }
1021         ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
1022
1023         if (args.depth > 0) {
1024                 struct cache_time mtime;
1025                 struct strbuf sb = STRBUF_INIT;
1026                 char *shallow = git_path("shallow");
1027                 int fd;
1028
1029                 mtime.sec = st.st_mtime;
1030                 mtime.nsec = ST_MTIME_NSEC(st);
1031                 if (stat(shallow, &st)) {
1032                         if (mtime.sec)
1033                                 die("shallow file was removed during fetch");
1034                 } else if (st.st_mtime != mtime.sec
1035 #ifdef USE_NSEC
1036                                 || ST_MTIME_NSEC(st) != mtime.nsec
1037 #endif
1038                           )
1039                         die("shallow file was changed during fetch");
1040
1041                 fd = hold_lock_file_for_update(&lock, shallow,
1042                                                LOCK_DIE_ON_ERROR);
1043                 if (!write_shallow_commits(&sb, 0)
1044                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1045                         unlink_or_warn(shallow);
1046                         rollback_lock_file(&lock);
1047                 } else {
1048                         commit_lock_file(&lock);
1049                 }
1050                 strbuf_release(&sb);
1051         }
1052
1053         reprepare_packed_git();
1054         return ref_cpy;
1055 }