Code

dd67e48fc7cb945d4f1e11f30abca89d16adfc9d
[git.git] / fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
10 static int keep_pack;
11 static int keep_auto;
12 static int quiet;
13 static int verbose;
14 static int fetch_all;
15 static int depth;
16 static const char fetch_pack_usage[] =
17 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [-v] [<host>:]<directory> [<refs>...]";
18 static const char *uploadpack = "git-upload-pack";
20 #define COMPLETE        (1U << 0)
21 #define COMMON          (1U << 1)
22 #define COMMON_REF      (1U << 2)
23 #define SEEN            (1U << 3)
24 #define POPPED          (1U << 4)
26 /*
27  * After sending this many "have"s if we do not get any new ACK , we
28  * give up traversing our history.
29  */
30 #define MAX_IN_VAIN 256
32 static struct commit_list *rev_list;
33 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
35 static void rev_list_push(struct commit *commit, int mark)
36 {
37         if (!(commit->object.flags & mark)) {
38                 commit->object.flags |= mark;
40                 if (!(commit->object.parsed))
41                         parse_commit(commit);
43                 insert_by_date(commit, &rev_list);
45                 if (!(commit->object.flags & COMMON))
46                         non_common_revs++;
47         }
48 }
50 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
51 {
52         struct object *o = deref_tag(parse_object(sha1), path, 0);
54         if (o && o->type == OBJ_COMMIT)
55                 rev_list_push((struct commit *)o, SEEN);
57         return 0;
58 }
60 /*
61    This function marks a rev and its ancestors as common.
62    In some cases, it is desirable to mark only the ancestors (for example
63    when only the server does not yet know that they are common).
64 */
66 static void mark_common(struct commit *commit,
67                 int ancestors_only, int dont_parse)
68 {
69         if (commit != NULL && !(commit->object.flags & COMMON)) {
70                 struct object *o = (struct object *)commit;
72                 if (!ancestors_only)
73                         o->flags |= COMMON;
75                 if (!(o->flags & SEEN))
76                         rev_list_push(commit, SEEN);
77                 else {
78                         struct commit_list *parents;
80                         if (!ancestors_only && !(o->flags & POPPED))
81                                 non_common_revs--;
82                         if (!o->parsed && !dont_parse)
83                                 parse_commit(commit);
85                         for (parents = commit->parents;
86                                         parents;
87                                         parents = parents->next)
88                                 mark_common(parents->item, 0, dont_parse);
89                 }
90         }
91 }
93 /*
94   Get the next rev to send, ignoring the common.
95 */
97 static const unsigned char* get_rev(void)
98 {
99         struct commit *commit = NULL;
101         while (commit == NULL) {
102                 unsigned int mark;
103                 struct commit_list* parents;
105                 if (rev_list == NULL || non_common_revs == 0)
106                         return NULL;
108                 commit = rev_list->item;
109                 if (!(commit->object.parsed))
110                         parse_commit(commit);
111                 commit->object.flags |= POPPED;
112                 if (!(commit->object.flags & COMMON))
113                         non_common_revs--;
114         
115                 parents = commit->parents;
117                 if (commit->object.flags & COMMON) {
118                         /* do not send "have", and ignore ancestors */
119                         commit = NULL;
120                         mark = COMMON | SEEN;
121                 } else if (commit->object.flags & COMMON_REF)
122                         /* send "have", and ignore ancestors */
123                         mark = COMMON | SEEN;
124                 else
125                         /* send "have", also for its ancestors */
126                         mark = SEEN;
128                 while (parents) {
129                         if (!(parents->item->object.flags & SEEN))
130                                 rev_list_push(parents->item, mark);
131                         if (mark & COMMON)
132                                 mark_common(parents->item, 1, 0);
133                         parents = parents->next;
134                 }
136                 rev_list = rev_list->next;
137         }
139         return commit->object.sha1;
142 static int find_common(int fd[2], unsigned char *result_sha1,
143                        struct ref *refs)
145         int fetching;
146         int count = 0, flushes = 0, retval;
147         const unsigned char *sha1;
148         unsigned in_vain = 0;
149         int got_continue = 0;
151         for_each_ref(rev_list_insert_ref, NULL);
153         fetching = 0;
154         for ( ; refs ; refs = refs->next) {
155                 unsigned char *remote = refs->old_sha1;
156                 struct object *o;
158                 /*
159                  * If that object is complete (i.e. it is an ancestor of a
160                  * local ref), we tell them we have it but do not have to
161                  * tell them about its ancestors, which they already know
162                  * about.
163                  *
164                  * We use lookup_object here because we are only
165                  * interested in the case we *know* the object is
166                  * reachable and we have already scanned it.
167                  */
168                 if (((o = lookup_object(remote)) != NULL) &&
169                                 (o->flags & COMPLETE)) {
170                         continue;
171                 }
173                 if (!fetching)
174                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
175                                      sha1_to_hex(remote),
176                                      (multi_ack ? " multi_ack" : ""),
177                                      (use_sideband == 2 ? " side-band-64k" : ""),
178                                      (use_sideband == 1 ? " side-band" : ""),
179                                      (use_thin_pack ? " thin-pack" : ""),
180                                      " ofs-delta");
181                 else
182                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
183                 fetching++;
184         }
185         if (is_repository_shallow())
186                 write_shallow_commits(fd[1], 1);
187         if (depth > 0)
188                 packet_write(fd[1], "deepen %d", depth);
189         packet_flush(fd[1]);
190         if (!fetching)
191                 return 1;
193         if (depth > 0) {
194                 char line[1024];
195                 unsigned char sha1[20];
196                 int len;
198                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
199                         if (!strncmp("shallow ", line, 8)) {
200                                 if (get_sha1_hex(line + 8, sha1))
201                                         die("invalid shallow line: %s", line);
202                                 register_shallow(sha1);
203                                 continue;
204                         }
205                         if (!strncmp("unshallow ", line, 10)) {
206                                 if (get_sha1_hex(line + 10, sha1))
207                                         die("invalid unshallow line: %s", line);
208                                 if (!lookup_object(sha1))
209                                         die("object not found: %s", line);
210                                 /* make sure that it is parsed as shallow */
211                                 parse_object(sha1);
212                                 if (unregister_shallow(sha1))
213                                         die("no shallow found: %s", line);
214                                 continue;
215                         }
216                         die("expected shallow/unshallow, got %s", line);
217                 }
218         }
220         flushes = 0;
221         retval = -1;
222         while ((sha1 = get_rev())) {
223                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
224                 if (verbose)
225                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
226                 in_vain++;
227                 if (!(31 & ++count)) {
228                         int ack;
230                         packet_flush(fd[1]);
231                         flushes++;
233                         /*
234                          * We keep one window "ahead" of the other side, and
235                          * will wait for an ACK only on the next one
236                          */
237                         if (count == 32)
238                                 continue;
240                         do {
241                                 ack = get_ack(fd[0], result_sha1);
242                                 if (verbose && ack)
243                                         fprintf(stderr, "got ack %d %s\n", ack,
244                                                         sha1_to_hex(result_sha1));
245                                 if (ack == 1) {
246                                         flushes = 0;
247                                         multi_ack = 0;
248                                         retval = 0;
249                                         goto done;
250                                 } else if (ack == 2) {
251                                         struct commit *commit =
252                                                 lookup_commit(result_sha1);
253                                         mark_common(commit, 0, 1);
254                                         retval = 0;
255                                         in_vain = 0;
256                                         got_continue = 1;
257                                 }
258                         } while (ack);
259                         flushes--;
260                         if (got_continue && MAX_IN_VAIN < in_vain) {
261                                 if (verbose)
262                                         fprintf(stderr, "giving up\n");
263                                 break; /* give up */
264                         }
265                 }
266         }
267 done:
268         packet_write(fd[1], "done\n");
269         if (verbose)
270                 fprintf(stderr, "done\n");
271         if (retval != 0) {
272                 multi_ack = 0;
273                 flushes++;
274         }
275         while (flushes || multi_ack) {
276                 int ack = get_ack(fd[0], result_sha1);
277                 if (ack) {
278                         if (verbose)
279                                 fprintf(stderr, "got ack (%d) %s\n", ack,
280                                         sha1_to_hex(result_sha1));
281                         if (ack == 1)
282                                 return 0;
283                         multi_ack = 1;
284                         continue;
285                 }
286                 flushes--;
287         }
288         return retval;
291 static struct commit_list *complete;
293 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
295         struct object *o = parse_object(sha1);
297         while (o && o->type == OBJ_TAG) {
298                 struct tag *t = (struct tag *) o;
299                 if (!t->tagged)
300                         break; /* broken repository */
301                 o->flags |= COMPLETE;
302                 o = parse_object(t->tagged->sha1);
303         }
304         if (o && o->type == OBJ_COMMIT) {
305                 struct commit *commit = (struct commit *)o;
306                 commit->object.flags |= COMPLETE;
307                 insert_by_date(commit, &complete);
308         }
309         return 0;
312 static void mark_recent_complete_commits(unsigned long cutoff)
314         while (complete && cutoff <= complete->item->date) {
315                 if (verbose)
316                         fprintf(stderr, "Marking %s as complete\n",
317                                 sha1_to_hex(complete->item->object.sha1));
318                 pop_most_recent_commit(&complete, COMPLETE);
319         }
322 static void filter_refs(struct ref **refs, int nr_match, char **match)
324         struct ref **return_refs;
325         struct ref *newlist = NULL;
326         struct ref **newtail = &newlist;
327         struct ref *ref, *next;
328         struct ref *fastarray[32];
330         if (nr_match && !fetch_all) {
331                 if (ARRAY_SIZE(fastarray) < nr_match)
332                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
333                 else {
334                         return_refs = fastarray;
335                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
336                 }
337         }
338         else
339                 return_refs = NULL;
341         for (ref = *refs; ref; ref = next) {
342                 next = ref->next;
343                 if (!memcmp(ref->name, "refs/", 5) &&
344                     check_ref_format(ref->name + 5))
345                         ; /* trash */
346                 else if (fetch_all &&
347                          (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
348                         *newtail = ref;
349                         ref->next = NULL;
350                         newtail = &ref->next;
351                         continue;
352                 }
353                 else {
354                         int order = path_match(ref->name, nr_match, match);
355                         if (order) {
356                                 return_refs[order-1] = ref;
357                                 continue; /* we will link it later */
358                         }
359                 }
360                 free(ref);
361         }
363         if (!fetch_all) {
364                 int i;
365                 for (i = 0; i < nr_match; i++) {
366                         ref = return_refs[i];
367                         if (ref) {
368                                 *newtail = ref;
369                                 ref->next = NULL;
370                                 newtail = &ref->next;
371                         }
372                 }
373                 if (return_refs != fastarray)
374                         free(return_refs);
375         }
376         *refs = newlist;
379 static int everything_local(struct ref **refs, int nr_match, char **match)
381         struct ref *ref;
382         int retval;
383         unsigned long cutoff = 0;
385         track_object_refs = 0;
386         save_commit_buffer = 0;
388         for (ref = *refs; ref; ref = ref->next) {
389                 struct object *o;
391                 o = parse_object(ref->old_sha1);
392                 if (!o)
393                         continue;
395                 /* We already have it -- which may mean that we were
396                  * in sync with the other side at some time after
397                  * that (it is OK if we guess wrong here).
398                  */
399                 if (o->type == OBJ_COMMIT) {
400                         struct commit *commit = (struct commit *)o;
401                         if (!cutoff || cutoff < commit->date)
402                                 cutoff = commit->date;
403                 }
404         }
406         if (!depth) {
407                 for_each_ref(mark_complete, NULL);
408                 if (cutoff)
409                         mark_recent_complete_commits(cutoff);
410         }
412         /*
413          * Mark all complete remote refs as common refs.
414          * Don't mark them common yet; the server has to be told so first.
415          */
416         for (ref = *refs; ref; ref = ref->next) {
417                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
418                                              NULL, 0);
420                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
421                         continue;
423                 if (!(o->flags & SEEN)) {
424                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
426                         mark_common((struct commit *)o, 1, 1);
427                 }
428         }
430         filter_refs(refs, nr_match, match);
432         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
433                 const unsigned char *remote = ref->old_sha1;
434                 unsigned char local[20];
435                 struct object *o;
437                 o = lookup_object(remote);
438                 if (!o || !(o->flags & COMPLETE)) {
439                         retval = 0;
440                         if (!verbose)
441                                 continue;
442                         fprintf(stderr,
443                                 "want %s (%s)\n", sha1_to_hex(remote),
444                                 ref->name);
445                         continue;
446                 }
448                 hashcpy(ref->new_sha1, local);
449                 if (!verbose)
450                         continue;
451                 fprintf(stderr,
452                         "already have %s (%s)\n", sha1_to_hex(remote),
453                         ref->name);
454         }
455         return retval;
458 static pid_t setup_sideband(int fd[2], int xd[2])
460         pid_t side_pid;
462         if (!use_sideband) {
463                 fd[0] = xd[0];
464                 fd[1] = xd[1];
465                 return 0;
466         }
467         /* xd[] is talking with upload-pack; subprocess reads from
468          * xd[0], spits out band#2 to stderr, and feeds us band#1
469          * through our fd[0].
470          */
471         if (pipe(fd) < 0)
472                 die("fetch-pack: unable to set up pipe");
473         side_pid = fork();
474         if (side_pid < 0)
475                 die("fetch-pack: unable to fork off sideband demultiplexer");
476         if (!side_pid) {
477                 /* subprocess */
478                 close(fd[0]);
479                 if (xd[0] != xd[1])
480                         close(xd[1]);
481                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
482                         exit(1);
483                 exit(0);
484         }
485         close(xd[0]);
486         close(fd[1]);
487         fd[1] = xd[1];
488         return side_pid;
491 static int get_pack(int xd[2])
493         int status;
494         pid_t pid, side_pid;
495         int fd[2];
496         const char *argv[20];
497         char keep_arg[256];
498         char hdr_arg[256];
499         const char **av;
500         int do_keep = keep_pack;
502         side_pid = setup_sideband(fd, xd);
504         av = argv;
505         *hdr_arg = 0;
506         if (keep_auto) {
507                 struct pack_header header;
509                 if (read_pack_header(fd[0], &header))
510                         die("protocol error: bad pack header");
511                 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
512                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
513                 if (ntohl(header.hdr_entries) < keep_auto)
514                         do_keep = 0;
515                 else
516                         do_keep = 1;
517         }
519         if (do_keep) {
520                 *av++ = "index-pack";
521                 *av++ = "--stdin";
522                 if (!quiet)
523                         *av++ = "-v";
524                 if (use_thin_pack)
525                         *av++ = "--fix-thin";
526                 if (keep_pack > 1 || keep_auto) {
527                         int s = sprintf(keep_arg,
528                                         "--keep=fetch-pack %d on ", getpid());
529                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
530                                 strcpy(keep_arg + s, "localhost");
531                         *av++ = keep_arg;
532                 }
533         }
534         else {
535                 *av++ = "unpack-objects";
536                 if (quiet)
537                         *av++ = "-q";
538         }
539         if (*hdr_arg)
540                 *av++ = hdr_arg;
541         *av++ = NULL;
543         pid = fork();
544         if (pid < 0)
545                 die("fetch-pack: unable to fork off %s", argv[0]);
546         if (!pid) {
547                 dup2(fd[0], 0);
548                 close(fd[0]);
549                 close(fd[1]);
550                 execv_git_cmd(argv);
551                 die("%s exec failed", argv[0]);
552         }
553         close(fd[0]);
554         close(fd[1]);
555         while (waitpid(pid, &status, 0) < 0) {
556                 if (errno != EINTR)
557                         die("waiting for %s: %s", argv[0], strerror(errno));
558         }
559         if (WIFEXITED(status)) {
560                 int code = WEXITSTATUS(status);
561                 if (code)
562                         die("%s died with error code %d", argv[0], code);
563                 return 0;
564         }
565         if (WIFSIGNALED(status)) {
566                 int sig = WTERMSIG(status);
567                 die("%s died of signal %d", argv[0], sig);
568         }
569         die("%s died of unnatural causes %d", argv[0], status);
572 static int fetch_pack(int fd[2], int nr_match, char **match)
574         struct ref *ref;
575         unsigned char sha1[20];
577         get_remote_heads(fd[0], &ref, 0, NULL, 0);
578         if (is_repository_shallow() && !server_supports("shallow"))
579                 die("Server does not support shallow clients");
580         if (server_supports("multi_ack")) {
581                 if (verbose)
582                         fprintf(stderr, "Server supports multi_ack\n");
583                 multi_ack = 1;
584         }
585         if (server_supports("side-band-64k")) {
586                 if (verbose)
587                         fprintf(stderr, "Server supports side-band-64k\n");
588                 use_sideband = 2;
589         }
590         else if (server_supports("side-band")) {
591                 if (verbose)
592                         fprintf(stderr, "Server supports side-band\n");
593                 use_sideband = 1;
594         }
595         if (!ref) {
596                 packet_flush(fd[1]);
597                 die("no matching remote head");
598         }
599         if (everything_local(&ref, nr_match, match)) {
600                 packet_flush(fd[1]);
601                 goto all_done;
602         }
603         if (find_common(fd, sha1, ref) < 0)
604                 if (keep_pack != 1)
605                         /* When cloning, it is not unusual to have
606                          * no common commit.
607                          */
608                         fprintf(stderr, "warning: no common commits\n");
610         if (get_pack(fd))
611                 die("git-fetch-pack: fetch failed.");
613  all_done:
614         while (ref) {
615                 printf("%s %s\n",
616                        sha1_to_hex(ref->old_sha1), ref->name);
617                 ref = ref->next;
618         }
619         return 0;
622 static int remove_duplicates(int nr_heads, char **heads)
624         int src, dst;
626         for (src = dst = 0; src < nr_heads; src++) {
627                 /* If heads[src] is different from any of
628                  * heads[0..dst], push it in.
629                  */
630                 int i;
631                 for (i = 0; i < dst; i++) {
632                         if (!strcmp(heads[i], heads[src]))
633                                 break;
634                 }
635                 if (i < dst)
636                         continue;
637                 if (src != dst)
638                         heads[dst] = heads[src];
639                 dst++;
640         }
641         heads[dst] = 0;
642         return dst;
645 static struct lock_file lock;
647 int main(int argc, char **argv)
649         int i, ret, nr_heads;
650         char *dest = NULL, **heads;
651         int fd[2];
652         pid_t pid;
653         struct stat st;
655         setup_git_directory();
657         nr_heads = 0;
658         heads = NULL;
659         for (i = 1; i < argc; i++) {
660                 char *arg = argv[i];
662                 if (*arg == '-') {
663                         if (!strncmp("--upload-pack=", arg, 14)) {
664                                 uploadpack = arg + 14;
665                                 continue;
666                         }
667                         if (!strncmp("--exec=", arg, 7)) {
668                                 uploadpack = arg + 7;
669                                 continue;
670                         }
671                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
672                                 quiet = 1;
673                                 continue;
674                         }
675                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
676                                 keep_pack++;
677                                 continue;
678                         }
679                         if (!strcmp("--keep-auto", arg)) {
680                                 keep_auto = 100;
681                                 continue;
682                         }
683                         if (!strncmp("--keep-auto=", arg, 12)) {
684                                 keep_auto = strtoul(arg + 12, NULL, 0);
685                                 if (keep_auto < 20)
686                                         keep_auto = 20;
687                                 continue;
688                         }
689                         if (!strcmp("--thin", arg)) {
690                                 use_thin_pack = 1;
691                                 continue;
692                         }
693                         if (!strcmp("--all", arg)) {
694                                 fetch_all = 1;
695                                 continue;
696                         }
697                         if (!strcmp("-v", arg)) {
698                                 verbose = 1;
699                                 continue;
700                         }
701                         if (!strncmp("--depth=", arg, 8)) {
702                                 depth = strtol(arg + 8, NULL, 0);
703                                 if (stat(git_path("shallow"), &st))
704                                         st.st_mtime = 0;
705                                 continue;
706                         }
707                         usage(fetch_pack_usage);
708                 }
709                 dest = arg;
710                 heads = argv + i + 1;
711                 nr_heads = argc - i - 1;
712                 break;
713         }
714         if (!dest)
715                 usage(fetch_pack_usage);
716         pid = git_connect(fd, dest, uploadpack);
717         if (pid < 0)
718                 return 1;
719         if (heads && nr_heads)
720                 nr_heads = remove_duplicates(nr_heads, heads);
721         ret = fetch_pack(fd, nr_heads, heads);
722         close(fd[0]);
723         close(fd[1]);
724         ret |= finish_connect(pid);
726         if (!ret && nr_heads) {
727                 /* If the heads to pull were given, we should have
728                  * consumed all of them by matching the remote.
729                  * Otherwise, 'git-fetch remote no-such-ref' would
730                  * silently succeed without issuing an error.
731                  */
732                 for (i = 0; i < nr_heads; i++)
733                         if (heads[i] && heads[i][0]) {
734                                 error("no such remote ref %s", heads[i]);
735                                 ret = 1;
736                         }
737         }
739         if (!ret && depth > 0) {
740                 struct cache_time mtime;
741                 char *shallow = git_path("shallow");
742                 int fd;
744                 mtime.sec = st.st_mtime;
745 #ifdef USE_NSEC
746                 mtime.usec = st.st_mtim.usec;
747 #endif
748                 if (stat(shallow, &st)) {
749                         if (mtime.sec)
750                                 die("shallow file was removed during fetch");
751                 } else if (st.st_mtime != mtime.sec
752 #ifdef USE_NSEC
753                                 || st.st_mtim.usec != mtime.usec
754 #endif
755                           )
756                         die("shallow file was changed during fetch");
758                 fd = hold_lock_file_for_update(&lock, shallow, 1);
759                 if (!write_shallow_commits(fd, 0)) {
760                         unlink(shallow);
761                         rollback_lock_file(&lock);
762                 } else {
763                         close(fd);
764                         commit_lock_file(&lock);
765                 }
766         }
768         return !!ret;