Code

fc0534ce03a69f45671d009772865cbfcad3b026
[git.git] / fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
10 static int keep_pack;
11 static int unpack_limit = 100;
12 static int quiet;
13 static int verbose;
14 static int fetch_all;
15 static int depth;
16 static const char fetch_pack_usage[] =
17 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [-v] [<host>:]<directory> [<refs>...]";
18 static const char *uploadpack = "git-upload-pack";
20 #define COMPLETE        (1U << 0)
21 #define COMMON          (1U << 1)
22 #define COMMON_REF      (1U << 2)
23 #define SEEN            (1U << 3)
24 #define POPPED          (1U << 4)
26 /*
27  * After sending this many "have"s if we do not get any new ACK , we
28  * give up traversing our history.
29  */
30 #define MAX_IN_VAIN 256
32 static struct commit_list *rev_list;
33 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
35 static void rev_list_push(struct commit *commit, int mark)
36 {
37         if (!(commit->object.flags & mark)) {
38                 commit->object.flags |= mark;
40                 if (!(commit->object.parsed))
41                         parse_commit(commit);
43                 insert_by_date(commit, &rev_list);
45                 if (!(commit->object.flags & COMMON))
46                         non_common_revs++;
47         }
48 }
50 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
51 {
52         struct object *o = deref_tag(parse_object(sha1), path, 0);
54         if (o && o->type == OBJ_COMMIT)
55                 rev_list_push((struct commit *)o, SEEN);
57         return 0;
58 }
60 /*
61    This function marks a rev and its ancestors as common.
62    In some cases, it is desirable to mark only the ancestors (for example
63    when only the server does not yet know that they are common).
64 */
66 static void mark_common(struct commit *commit,
67                 int ancestors_only, int dont_parse)
68 {
69         if (commit != NULL && !(commit->object.flags & COMMON)) {
70                 struct object *o = (struct object *)commit;
72                 if (!ancestors_only)
73                         o->flags |= COMMON;
75                 if (!(o->flags & SEEN))
76                         rev_list_push(commit, SEEN);
77                 else {
78                         struct commit_list *parents;
80                         if (!ancestors_only && !(o->flags & POPPED))
81                                 non_common_revs--;
82                         if (!o->parsed && !dont_parse)
83                                 parse_commit(commit);
85                         for (parents = commit->parents;
86                                         parents;
87                                         parents = parents->next)
88                                 mark_common(parents->item, 0, dont_parse);
89                 }
90         }
91 }
93 /*
94   Get the next rev to send, ignoring the common.
95 */
97 static const unsigned char* get_rev(void)
98 {
99         struct commit *commit = NULL;
101         while (commit == NULL) {
102                 unsigned int mark;
103                 struct commit_list* parents;
105                 if (rev_list == NULL || non_common_revs == 0)
106                         return NULL;
108                 commit = rev_list->item;
109                 if (!(commit->object.parsed))
110                         parse_commit(commit);
111                 commit->object.flags |= POPPED;
112                 if (!(commit->object.flags & COMMON))
113                         non_common_revs--;
114         
115                 parents = commit->parents;
117                 if (commit->object.flags & COMMON) {
118                         /* do not send "have", and ignore ancestors */
119                         commit = NULL;
120                         mark = COMMON | SEEN;
121                 } else if (commit->object.flags & COMMON_REF)
122                         /* send "have", and ignore ancestors */
123                         mark = COMMON | SEEN;
124                 else
125                         /* send "have", also for its ancestors */
126                         mark = SEEN;
128                 while (parents) {
129                         if (!(parents->item->object.flags & SEEN))
130                                 rev_list_push(parents->item, mark);
131                         if (mark & COMMON)
132                                 mark_common(parents->item, 1, 0);
133                         parents = parents->next;
134                 }
136                 rev_list = rev_list->next;
137         }
139         return commit->object.sha1;
142 static int find_common(int fd[2], unsigned char *result_sha1,
143                        struct ref *refs)
145         int fetching;
146         int count = 0, flushes = 0, retval;
147         const unsigned char *sha1;
148         unsigned in_vain = 0;
149         int got_continue = 0;
151         for_each_ref(rev_list_insert_ref, NULL);
153         fetching = 0;
154         for ( ; refs ; refs = refs->next) {
155                 unsigned char *remote = refs->old_sha1;
156                 struct object *o;
158                 /*
159                  * If that object is complete (i.e. it is an ancestor of a
160                  * local ref), we tell them we have it but do not have to
161                  * tell them about its ancestors, which they already know
162                  * about.
163                  *
164                  * We use lookup_object here because we are only
165                  * interested in the case we *know* the object is
166                  * reachable and we have already scanned it.
167                  */
168                 if (((o = lookup_object(remote)) != NULL) &&
169                                 (o->flags & COMPLETE)) {
170                         continue;
171                 }
173                 if (!fetching)
174                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
175                                      sha1_to_hex(remote),
176                                      (multi_ack ? " multi_ack" : ""),
177                                      (use_sideband == 2 ? " side-band-64k" : ""),
178                                      (use_sideband == 1 ? " side-band" : ""),
179                                      (use_thin_pack ? " thin-pack" : ""),
180                                      " ofs-delta");
181                 else
182                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
183                 fetching++;
184         }
185         if (is_repository_shallow())
186                 write_shallow_commits(fd[1], 1);
187         if (depth > 0)
188                 packet_write(fd[1], "deepen %d", depth);
189         packet_flush(fd[1]);
190         if (!fetching)
191                 return 1;
193         if (depth > 0) {
194                 char line[1024];
195                 unsigned char sha1[20];
196                 int len;
198                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
199                         if (!strncmp("shallow ", line, 8)) {
200                                 if (get_sha1_hex(line + 8, sha1))
201                                         die("invalid shallow line: %s", line);
202                                 register_shallow(sha1);
203                                 continue;
204                         }
205                         if (!strncmp("unshallow ", line, 10)) {
206                                 if (get_sha1_hex(line + 10, sha1))
207                                         die("invalid unshallow line: %s", line);
208                                 if (!lookup_object(sha1))
209                                         die("object not found: %s", line);
210                                 /* make sure that it is parsed as shallow */
211                                 parse_object(sha1);
212                                 if (unregister_shallow(sha1))
213                                         die("no shallow found: %s", line);
214                                 continue;
215                         }
216                         die("expected shallow/unshallow, got %s", line);
217                 }
218         }
220         flushes = 0;
221         retval = -1;
222         while ((sha1 = get_rev())) {
223                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
224                 if (verbose)
225                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
226                 in_vain++;
227                 if (!(31 & ++count)) {
228                         int ack;
230                         packet_flush(fd[1]);
231                         flushes++;
233                         /*
234                          * We keep one window "ahead" of the other side, and
235                          * will wait for an ACK only on the next one
236                          */
237                         if (count == 32)
238                                 continue;
240                         do {
241                                 ack = get_ack(fd[0], result_sha1);
242                                 if (verbose && ack)
243                                         fprintf(stderr, "got ack %d %s\n", ack,
244                                                         sha1_to_hex(result_sha1));
245                                 if (ack == 1) {
246                                         flushes = 0;
247                                         multi_ack = 0;
248                                         retval = 0;
249                                         goto done;
250                                 } else if (ack == 2) {
251                                         struct commit *commit =
252                                                 lookup_commit(result_sha1);
253                                         mark_common(commit, 0, 1);
254                                         retval = 0;
255                                         in_vain = 0;
256                                         got_continue = 1;
257                                 }
258                         } while (ack);
259                         flushes--;
260                         if (got_continue && MAX_IN_VAIN < in_vain) {
261                                 if (verbose)
262                                         fprintf(stderr, "giving up\n");
263                                 break; /* give up */
264                         }
265                 }
266         }
267 done:
268         packet_write(fd[1], "done\n");
269         if (verbose)
270                 fprintf(stderr, "done\n");
271         if (retval != 0) {
272                 multi_ack = 0;
273                 flushes++;
274         }
275         while (flushes || multi_ack) {
276                 int ack = get_ack(fd[0], result_sha1);
277                 if (ack) {
278                         if (verbose)
279                                 fprintf(stderr, "got ack (%d) %s\n", ack,
280                                         sha1_to_hex(result_sha1));
281                         if (ack == 1)
282                                 return 0;
283                         multi_ack = 1;
284                         continue;
285                 }
286                 flushes--;
287         }
288         return retval;
291 static struct commit_list *complete;
293 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
295         struct object *o = parse_object(sha1);
297         while (o && o->type == OBJ_TAG) {
298                 struct tag *t = (struct tag *) o;
299                 if (!t->tagged)
300                         break; /* broken repository */
301                 o->flags |= COMPLETE;
302                 o = parse_object(t->tagged->sha1);
303         }
304         if (o && o->type == OBJ_COMMIT) {
305                 struct commit *commit = (struct commit *)o;
306                 commit->object.flags |= COMPLETE;
307                 insert_by_date(commit, &complete);
308         }
309         return 0;
312 static void mark_recent_complete_commits(unsigned long cutoff)
314         while (complete && cutoff <= complete->item->date) {
315                 if (verbose)
316                         fprintf(stderr, "Marking %s as complete\n",
317                                 sha1_to_hex(complete->item->object.sha1));
318                 pop_most_recent_commit(&complete, COMPLETE);
319         }
322 static void filter_refs(struct ref **refs, int nr_match, char **match)
324         struct ref **return_refs;
325         struct ref *newlist = NULL;
326         struct ref **newtail = &newlist;
327         struct ref *ref, *next;
328         struct ref *fastarray[32];
330         if (nr_match && !fetch_all) {
331                 if (ARRAY_SIZE(fastarray) < nr_match)
332                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
333                 else {
334                         return_refs = fastarray;
335                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
336                 }
337         }
338         else
339                 return_refs = NULL;
341         for (ref = *refs; ref; ref = next) {
342                 next = ref->next;
343                 if (!memcmp(ref->name, "refs/", 5) &&
344                     check_ref_format(ref->name + 5))
345                         ; /* trash */
346                 else if (fetch_all &&
347                          (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
348                         *newtail = ref;
349                         ref->next = NULL;
350                         newtail = &ref->next;
351                         continue;
352                 }
353                 else {
354                         int order = path_match(ref->name, nr_match, match);
355                         if (order) {
356                                 return_refs[order-1] = ref;
357                                 continue; /* we will link it later */
358                         }
359                 }
360                 free(ref);
361         }
363         if (!fetch_all) {
364                 int i;
365                 for (i = 0; i < nr_match; i++) {
366                         ref = return_refs[i];
367                         if (ref) {
368                                 *newtail = ref;
369                                 ref->next = NULL;
370                                 newtail = &ref->next;
371                         }
372                 }
373                 if (return_refs != fastarray)
374                         free(return_refs);
375         }
376         *refs = newlist;
379 static int everything_local(struct ref **refs, int nr_match, char **match)
381         struct ref *ref;
382         int retval;
383         unsigned long cutoff = 0;
385         track_object_refs = 0;
386         save_commit_buffer = 0;
388         for (ref = *refs; ref; ref = ref->next) {
389                 struct object *o;
391                 o = parse_object(ref->old_sha1);
392                 if (!o)
393                         continue;
395                 /* We already have it -- which may mean that we were
396                  * in sync with the other side at some time after
397                  * that (it is OK if we guess wrong here).
398                  */
399                 if (o->type == OBJ_COMMIT) {
400                         struct commit *commit = (struct commit *)o;
401                         if (!cutoff || cutoff < commit->date)
402                                 cutoff = commit->date;
403                 }
404         }
406         if (!depth) {
407                 for_each_ref(mark_complete, NULL);
408                 if (cutoff)
409                         mark_recent_complete_commits(cutoff);
410         }
412         /*
413          * Mark all complete remote refs as common refs.
414          * Don't mark them common yet; the server has to be told so first.
415          */
416         for (ref = *refs; ref; ref = ref->next) {
417                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
418                                              NULL, 0);
420                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
421                         continue;
423                 if (!(o->flags & SEEN)) {
424                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
426                         mark_common((struct commit *)o, 1, 1);
427                 }
428         }
430         filter_refs(refs, nr_match, match);
432         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
433                 const unsigned char *remote = ref->old_sha1;
434                 unsigned char local[20];
435                 struct object *o;
437                 o = lookup_object(remote);
438                 if (!o || !(o->flags & COMPLETE)) {
439                         retval = 0;
440                         if (!verbose)
441                                 continue;
442                         fprintf(stderr,
443                                 "want %s (%s)\n", sha1_to_hex(remote),
444                                 ref->name);
445                         continue;
446                 }
448                 hashcpy(ref->new_sha1, local);
449                 if (!verbose)
450                         continue;
451                 fprintf(stderr,
452                         "already have %s (%s)\n", sha1_to_hex(remote),
453                         ref->name);
454         }
455         return retval;
458 static pid_t setup_sideband(int fd[2], int xd[2])
460         pid_t side_pid;
462         if (!use_sideband) {
463                 fd[0] = xd[0];
464                 fd[1] = xd[1];
465                 return 0;
466         }
467         /* xd[] is talking with upload-pack; subprocess reads from
468          * xd[0], spits out band#2 to stderr, and feeds us band#1
469          * through our fd[0].
470          */
471         if (pipe(fd) < 0)
472                 die("fetch-pack: unable to set up pipe");
473         side_pid = fork();
474         if (side_pid < 0)
475                 die("fetch-pack: unable to fork off sideband demultiplexer");
476         if (!side_pid) {
477                 /* subprocess */
478                 close(fd[0]);
479                 if (xd[0] != xd[1])
480                         close(xd[1]);
481                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
482                         exit(1);
483                 exit(0);
484         }
485         close(xd[0]);
486         close(fd[1]);
487         fd[1] = xd[1];
488         return side_pid;
491 static int get_pack(int xd[2])
493         int status;
494         pid_t pid, side_pid;
495         int fd[2];
496         const char *argv[20];
497         char keep_arg[256];
498         char hdr_arg[256];
499         const char **av;
500         int do_keep = keep_pack;
502         side_pid = setup_sideband(fd, xd);
504         av = argv;
505         *hdr_arg = 0;
506         if (unpack_limit) {
507                 struct pack_header header;
509                 if (read_pack_header(fd[0], &header))
510                         die("protocol error: bad pack header");
511                 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
512                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
513                 if (ntohl(header.hdr_entries) < unpack_limit)
514                         do_keep = 0;
515                 else
516                         do_keep = 1;
517         }
519         if (do_keep) {
520                 *av++ = "index-pack";
521                 *av++ = "--stdin";
522                 if (!quiet)
523                         *av++ = "-v";
524                 if (use_thin_pack)
525                         *av++ = "--fix-thin";
526                 if (keep_pack > 1 || unpack_limit) {
527                         int s = sprintf(keep_arg,
528                                         "--keep=fetch-pack %d on ", getpid());
529                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
530                                 strcpy(keep_arg + s, "localhost");
531                         *av++ = keep_arg;
532                 }
533         }
534         else {
535                 *av++ = "unpack-objects";
536                 if (quiet)
537                         *av++ = "-q";
538         }
539         if (*hdr_arg)
540                 *av++ = hdr_arg;
541         *av++ = NULL;
543         pid = fork();
544         if (pid < 0)
545                 die("fetch-pack: unable to fork off %s", argv[0]);
546         if (!pid) {
547                 dup2(fd[0], 0);
548                 close(fd[0]);
549                 close(fd[1]);
550                 execv_git_cmd(argv);
551                 die("%s exec failed", argv[0]);
552         }
553         close(fd[0]);
554         close(fd[1]);
555         while (waitpid(pid, &status, 0) < 0) {
556                 if (errno != EINTR)
557                         die("waiting for %s: %s", argv[0], strerror(errno));
558         }
559         if (WIFEXITED(status)) {
560                 int code = WEXITSTATUS(status);
561                 if (code)
562                         die("%s died with error code %d", argv[0], code);
563                 return 0;
564         }
565         if (WIFSIGNALED(status)) {
566                 int sig = WTERMSIG(status);
567                 die("%s died of signal %d", argv[0], sig);
568         }
569         die("%s died of unnatural causes %d", argv[0], status);
572 static int fetch_pack(int fd[2], int nr_match, char **match)
574         struct ref *ref;
575         unsigned char sha1[20];
577         get_remote_heads(fd[0], &ref, 0, NULL, 0);
578         if (is_repository_shallow() && !server_supports("shallow"))
579                 die("Server does not support shallow clients");
580         if (server_supports("multi_ack")) {
581                 if (verbose)
582                         fprintf(stderr, "Server supports multi_ack\n");
583                 multi_ack = 1;
584         }
585         if (server_supports("side-band-64k")) {
586                 if (verbose)
587                         fprintf(stderr, "Server supports side-band-64k\n");
588                 use_sideband = 2;
589         }
590         else if (server_supports("side-band")) {
591                 if (verbose)
592                         fprintf(stderr, "Server supports side-band\n");
593                 use_sideband = 1;
594         }
595         if (!ref) {
596                 packet_flush(fd[1]);
597                 die("no matching remote head");
598         }
599         if (everything_local(&ref, nr_match, match)) {
600                 packet_flush(fd[1]);
601                 goto all_done;
602         }
603         if (find_common(fd, sha1, ref) < 0)
604                 if (keep_pack != 1)
605                         /* When cloning, it is not unusual to have
606                          * no common commit.
607                          */
608                         fprintf(stderr, "warning: no common commits\n");
610         if (get_pack(fd))
611                 die("git-fetch-pack: fetch failed.");
613  all_done:
614         while (ref) {
615                 printf("%s %s\n",
616                        sha1_to_hex(ref->old_sha1), ref->name);
617                 ref = ref->next;
618         }
619         return 0;
622 static int remove_duplicates(int nr_heads, char **heads)
624         int src, dst;
626         for (src = dst = 0; src < nr_heads; src++) {
627                 /* If heads[src] is different from any of
628                  * heads[0..dst], push it in.
629                  */
630                 int i;
631                 for (i = 0; i < dst; i++) {
632                         if (!strcmp(heads[i], heads[src]))
633                                 break;
634                 }
635                 if (i < dst)
636                         continue;
637                 if (src != dst)
638                         heads[dst] = heads[src];
639                 dst++;
640         }
641         heads[dst] = 0;
642         return dst;
645 static int fetch_pack_config(const char *var, const char *value)
647         if (strcmp(var, "fetch.unpacklimit") == 0) {
648                 unpack_limit = git_config_int(var, value);
649                 return 0;
650         }
652         return git_default_config(var, value);
655 static struct lock_file lock;
657 int main(int argc, char **argv)
659         int i, ret, nr_heads;
660         char *dest = NULL, **heads;
661         int fd[2];
662         pid_t pid;
663         struct stat st;
665         setup_git_directory();
666         setup_ident();
667         git_config(fetch_pack_config);
669         nr_heads = 0;
670         heads = NULL;
671         for (i = 1; i < argc; i++) {
672                 char *arg = argv[i];
674                 if (*arg == '-') {
675                         if (!strncmp("--upload-pack=", arg, 14)) {
676                                 uploadpack = arg + 14;
677                                 continue;
678                         }
679                         if (!strncmp("--exec=", arg, 7)) {
680                                 uploadpack = arg + 7;
681                                 continue;
682                         }
683                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
684                                 quiet = 1;
685                                 continue;
686                         }
687                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
688                                 keep_pack++;
689                                 unpack_limit = 0;
690                                 continue;
691                         }
692                         if (!strcmp("--thin", arg)) {
693                                 use_thin_pack = 1;
694                                 continue;
695                         }
696                         if (!strcmp("--all", arg)) {
697                                 fetch_all = 1;
698                                 continue;
699                         }
700                         if (!strcmp("-v", arg)) {
701                                 verbose = 1;
702                                 continue;
703                         }
704                         if (!strncmp("--depth=", arg, 8)) {
705                                 depth = strtol(arg + 8, NULL, 0);
706                                 if (stat(git_path("shallow"), &st))
707                                         st.st_mtime = 0;
708                                 continue;
709                         }
710                         usage(fetch_pack_usage);
711                 }
712                 dest = arg;
713                 heads = argv + i + 1;
714                 nr_heads = argc - i - 1;
715                 break;
716         }
717         if (!dest)
718                 usage(fetch_pack_usage);
719         pid = git_connect(fd, dest, uploadpack);
720         if (pid < 0)
721                 return 1;
722         if (heads && nr_heads)
723                 nr_heads = remove_duplicates(nr_heads, heads);
724         ret = fetch_pack(fd, nr_heads, heads);
725         close(fd[0]);
726         close(fd[1]);
727         ret |= finish_connect(pid);
729         if (!ret && nr_heads) {
730                 /* If the heads to pull were given, we should have
731                  * consumed all of them by matching the remote.
732                  * Otherwise, 'git-fetch remote no-such-ref' would
733                  * silently succeed without issuing an error.
734                  */
735                 for (i = 0; i < nr_heads; i++)
736                         if (heads[i] && heads[i][0]) {
737                                 error("no such remote ref %s", heads[i]);
738                                 ret = 1;
739                         }
740         }
742         if (!ret && depth > 0) {
743                 struct cache_time mtime;
744                 char *shallow = git_path("shallow");
745                 int fd;
747                 mtime.sec = st.st_mtime;
748 #ifdef USE_NSEC
749                 mtime.usec = st.st_mtim.usec;
750 #endif
751                 if (stat(shallow, &st)) {
752                         if (mtime.sec)
753                                 die("shallow file was removed during fetch");
754                 } else if (st.st_mtime != mtime.sec
755 #ifdef USE_NSEC
756                                 || st.st_mtim.usec != mtime.usec
757 #endif
758                           )
759                         die("shallow file was changed during fetch");
761                 fd = hold_lock_file_for_update(&lock, shallow, 1);
762                 if (!write_shallow_commits(fd, 0)) {
763                         unlink(shallow);
764                         rollback_lock_file(&lock);
765                 } else {
766                         close(fd);
767                         commit_lock_file(&lock);
768                 }
769         }
771         return !!ret;