Code

fetch & clone: do not output progress when not on a tty
[git.git] / fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
10 static int keep_pack;
11 static int transfer_unpack_limit = -1;
12 static int fetch_unpack_limit = -1;
13 static int unpack_limit = 100;
14 static int quiet;
15 static int verbose;
16 static int fetch_all;
17 static int depth;
18 static int no_progress;
19 static const char fetch_pack_usage[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
21 static const char *uploadpack = "git-upload-pack";
23 #define COMPLETE        (1U << 0)
24 #define COMMON          (1U << 1)
25 #define COMMON_REF      (1U << 2)
26 #define SEEN            (1U << 3)
27 #define POPPED          (1U << 4)
29 /*
30  * After sending this many "have"s if we do not get any new ACK , we
31  * give up traversing our history.
32  */
33 #define MAX_IN_VAIN 256
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
38 static void rev_list_push(struct commit *commit, int mark)
39 {
40         if (!(commit->object.flags & mark)) {
41                 commit->object.flags |= mark;
43                 if (!(commit->object.parsed))
44                         parse_commit(commit);
46                 insert_by_date(commit, &rev_list);
48                 if (!(commit->object.flags & COMMON))
49                         non_common_revs++;
50         }
51 }
53 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
54 {
55         struct object *o = deref_tag(parse_object(sha1), path, 0);
57         if (o && o->type == OBJ_COMMIT)
58                 rev_list_push((struct commit *)o, SEEN);
60         return 0;
61 }
63 /*
64    This function marks a rev and its ancestors as common.
65    In some cases, it is desirable to mark only the ancestors (for example
66    when only the server does not yet know that they are common).
67 */
69 static void mark_common(struct commit *commit,
70                 int ancestors_only, int dont_parse)
71 {
72         if (commit != NULL && !(commit->object.flags & COMMON)) {
73                 struct object *o = (struct object *)commit;
75                 if (!ancestors_only)
76                         o->flags |= COMMON;
78                 if (!(o->flags & SEEN))
79                         rev_list_push(commit, SEEN);
80                 else {
81                         struct commit_list *parents;
83                         if (!ancestors_only && !(o->flags & POPPED))
84                                 non_common_revs--;
85                         if (!o->parsed && !dont_parse)
86                                 parse_commit(commit);
88                         for (parents = commit->parents;
89                                         parents;
90                                         parents = parents->next)
91                                 mark_common(parents->item, 0, dont_parse);
92                 }
93         }
94 }
96 /*
97   Get the next rev to send, ignoring the common.
98 */
100 static const unsigned char* get_rev(void)
102         struct commit *commit = NULL;
104         while (commit == NULL) {
105                 unsigned int mark;
106                 struct commit_list* parents;
108                 if (rev_list == NULL || non_common_revs == 0)
109                         return NULL;
111                 commit = rev_list->item;
112                 if (!(commit->object.parsed))
113                         parse_commit(commit);
114                 commit->object.flags |= POPPED;
115                 if (!(commit->object.flags & COMMON))
116                         non_common_revs--;
117         
118                 parents = commit->parents;
120                 if (commit->object.flags & COMMON) {
121                         /* do not send "have", and ignore ancestors */
122                         commit = NULL;
123                         mark = COMMON | SEEN;
124                 } else if (commit->object.flags & COMMON_REF)
125                         /* send "have", and ignore ancestors */
126                         mark = COMMON | SEEN;
127                 else
128                         /* send "have", also for its ancestors */
129                         mark = SEEN;
131                 while (parents) {
132                         if (!(parents->item->object.flags & SEEN))
133                                 rev_list_push(parents->item, mark);
134                         if (mark & COMMON)
135                                 mark_common(parents->item, 1, 0);
136                         parents = parents->next;
137                 }
139                 rev_list = rev_list->next;
140         }
142         return commit->object.sha1;
145 static int find_common(int fd[2], unsigned char *result_sha1,
146                        struct ref *refs)
148         int fetching;
149         int count = 0, flushes = 0, retval;
150         const unsigned char *sha1;
151         unsigned in_vain = 0;
152         int got_continue = 0;
154         for_each_ref(rev_list_insert_ref, NULL);
156         fetching = 0;
157         for ( ; refs ; refs = refs->next) {
158                 unsigned char *remote = refs->old_sha1;
159                 struct object *o;
161                 /*
162                  * If that object is complete (i.e. it is an ancestor of a
163                  * local ref), we tell them we have it but do not have to
164                  * tell them about its ancestors, which they already know
165                  * about.
166                  *
167                  * We use lookup_object here because we are only
168                  * interested in the case we *know* the object is
169                  * reachable and we have already scanned it.
170                  */
171                 if (((o = lookup_object(remote)) != NULL) &&
172                                 (o->flags & COMPLETE)) {
173                         continue;
174                 }
176                 if (!fetching)
177                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
178                                      sha1_to_hex(remote),
179                                      (multi_ack ? " multi_ack" : ""),
180                                      (use_sideband == 2 ? " side-band-64k" : ""),
181                                      (use_sideband == 1 ? " side-band" : ""),
182                                      (use_thin_pack ? " thin-pack" : ""),
183                                      " ofs-delta");
184                 else
185                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
186                 fetching++;
187         }
188         if (is_repository_shallow())
189                 write_shallow_commits(fd[1], 1);
190         if (depth > 0)
191                 packet_write(fd[1], "deepen %d", depth);
192         packet_flush(fd[1]);
193         if (!fetching)
194                 return 1;
196         if (depth > 0) {
197                 char line[1024];
198                 unsigned char sha1[20];
199                 int len;
201                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
202                         if (!strncmp("shallow ", line, 8)) {
203                                 if (get_sha1_hex(line + 8, sha1))
204                                         die("invalid shallow line: %s", line);
205                                 register_shallow(sha1);
206                                 continue;
207                         }
208                         if (!strncmp("unshallow ", line, 10)) {
209                                 if (get_sha1_hex(line + 10, sha1))
210                                         die("invalid unshallow line: %s", line);
211                                 if (!lookup_object(sha1))
212                                         die("object not found: %s", line);
213                                 /* make sure that it is parsed as shallow */
214                                 parse_object(sha1);
215                                 if (unregister_shallow(sha1))
216                                         die("no shallow found: %s", line);
217                                 continue;
218                         }
219                         die("expected shallow/unshallow, got %s", line);
220                 }
221         }
223         flushes = 0;
224         retval = -1;
225         while ((sha1 = get_rev())) {
226                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
227                 if (verbose)
228                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
229                 in_vain++;
230                 if (!(31 & ++count)) {
231                         int ack;
233                         packet_flush(fd[1]);
234                         flushes++;
236                         /*
237                          * We keep one window "ahead" of the other side, and
238                          * will wait for an ACK only on the next one
239                          */
240                         if (count == 32)
241                                 continue;
243                         do {
244                                 ack = get_ack(fd[0], result_sha1);
245                                 if (verbose && ack)
246                                         fprintf(stderr, "got ack %d %s\n", ack,
247                                                         sha1_to_hex(result_sha1));
248                                 if (ack == 1) {
249                                         flushes = 0;
250                                         multi_ack = 0;
251                                         retval = 0;
252                                         goto done;
253                                 } else if (ack == 2) {
254                                         struct commit *commit =
255                                                 lookup_commit(result_sha1);
256                                         mark_common(commit, 0, 1);
257                                         retval = 0;
258                                         in_vain = 0;
259                                         got_continue = 1;
260                                 }
261                         } while (ack);
262                         flushes--;
263                         if (got_continue && MAX_IN_VAIN < in_vain) {
264                                 if (verbose)
265                                         fprintf(stderr, "giving up\n");
266                                 break; /* give up */
267                         }
268                 }
269         }
270 done:
271         packet_write(fd[1], "done\n");
272         if (verbose)
273                 fprintf(stderr, "done\n");
274         if (retval != 0) {
275                 multi_ack = 0;
276                 flushes++;
277         }
278         while (flushes || multi_ack) {
279                 int ack = get_ack(fd[0], result_sha1);
280                 if (ack) {
281                         if (verbose)
282                                 fprintf(stderr, "got ack (%d) %s\n", ack,
283                                         sha1_to_hex(result_sha1));
284                         if (ack == 1)
285                                 return 0;
286                         multi_ack = 1;
287                         continue;
288                 }
289                 flushes--;
290         }
291         return retval;
294 static struct commit_list *complete;
296 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
298         struct object *o = parse_object(sha1);
300         while (o && o->type == OBJ_TAG) {
301                 struct tag *t = (struct tag *) o;
302                 if (!t->tagged)
303                         break; /* broken repository */
304                 o->flags |= COMPLETE;
305                 o = parse_object(t->tagged->sha1);
306         }
307         if (o && o->type == OBJ_COMMIT) {
308                 struct commit *commit = (struct commit *)o;
309                 commit->object.flags |= COMPLETE;
310                 insert_by_date(commit, &complete);
311         }
312         return 0;
315 static void mark_recent_complete_commits(unsigned long cutoff)
317         while (complete && cutoff <= complete->item->date) {
318                 if (verbose)
319                         fprintf(stderr, "Marking %s as complete\n",
320                                 sha1_to_hex(complete->item->object.sha1));
321                 pop_most_recent_commit(&complete, COMPLETE);
322         }
325 static void filter_refs(struct ref **refs, int nr_match, char **match)
327         struct ref **return_refs;
328         struct ref *newlist = NULL;
329         struct ref **newtail = &newlist;
330         struct ref *ref, *next;
331         struct ref *fastarray[32];
333         if (nr_match && !fetch_all) {
334                 if (ARRAY_SIZE(fastarray) < nr_match)
335                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
336                 else {
337                         return_refs = fastarray;
338                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
339                 }
340         }
341         else
342                 return_refs = NULL;
344         for (ref = *refs; ref; ref = next) {
345                 next = ref->next;
346                 if (!memcmp(ref->name, "refs/", 5) &&
347                     check_ref_format(ref->name + 5))
348                         ; /* trash */
349                 else if (fetch_all &&
350                          (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
351                         *newtail = ref;
352                         ref->next = NULL;
353                         newtail = &ref->next;
354                         continue;
355                 }
356                 else {
357                         int order = path_match(ref->name, nr_match, match);
358                         if (order) {
359                                 return_refs[order-1] = ref;
360                                 continue; /* we will link it later */
361                         }
362                 }
363                 free(ref);
364         }
366         if (!fetch_all) {
367                 int i;
368                 for (i = 0; i < nr_match; i++) {
369                         ref = return_refs[i];
370                         if (ref) {
371                                 *newtail = ref;
372                                 ref->next = NULL;
373                                 newtail = &ref->next;
374                         }
375                 }
376                 if (return_refs != fastarray)
377                         free(return_refs);
378         }
379         *refs = newlist;
382 static int everything_local(struct ref **refs, int nr_match, char **match)
384         struct ref *ref;
385         int retval;
386         unsigned long cutoff = 0;
388         track_object_refs = 0;
389         save_commit_buffer = 0;
391         for (ref = *refs; ref; ref = ref->next) {
392                 struct object *o;
394                 o = parse_object(ref->old_sha1);
395                 if (!o)
396                         continue;
398                 /* We already have it -- which may mean that we were
399                  * in sync with the other side at some time after
400                  * that (it is OK if we guess wrong here).
401                  */
402                 if (o->type == OBJ_COMMIT) {
403                         struct commit *commit = (struct commit *)o;
404                         if (!cutoff || cutoff < commit->date)
405                                 cutoff = commit->date;
406                 }
407         }
409         if (!depth) {
410                 for_each_ref(mark_complete, NULL);
411                 if (cutoff)
412                         mark_recent_complete_commits(cutoff);
413         }
415         /*
416          * Mark all complete remote refs as common refs.
417          * Don't mark them common yet; the server has to be told so first.
418          */
419         for (ref = *refs; ref; ref = ref->next) {
420                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
421                                              NULL, 0);
423                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
424                         continue;
426                 if (!(o->flags & SEEN)) {
427                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
429                         mark_common((struct commit *)o, 1, 1);
430                 }
431         }
433         filter_refs(refs, nr_match, match);
435         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
436                 const unsigned char *remote = ref->old_sha1;
437                 unsigned char local[20];
438                 struct object *o;
440                 o = lookup_object(remote);
441                 if (!o || !(o->flags & COMPLETE)) {
442                         retval = 0;
443                         if (!verbose)
444                                 continue;
445                         fprintf(stderr,
446                                 "want %s (%s)\n", sha1_to_hex(remote),
447                                 ref->name);
448                         continue;
449                 }
451                 hashcpy(ref->new_sha1, local);
452                 if (!verbose)
453                         continue;
454                 fprintf(stderr,
455                         "already have %s (%s)\n", sha1_to_hex(remote),
456                         ref->name);
457         }
458         return retval;
461 static pid_t setup_sideband(int fd[2], int xd[2])
463         pid_t side_pid;
465         if (!use_sideband) {
466                 fd[0] = xd[0];
467                 fd[1] = xd[1];
468                 return 0;
469         }
470         /* xd[] is talking with upload-pack; subprocess reads from
471          * xd[0], spits out band#2 to stderr, and feeds us band#1
472          * through our fd[0].
473          */
474         if (pipe(fd) < 0)
475                 die("fetch-pack: unable to set up pipe");
476         side_pid = fork();
477         if (side_pid < 0)
478                 die("fetch-pack: unable to fork off sideband demultiplexer");
479         if (!side_pid) {
480                 /* subprocess */
481                 close(fd[0]);
482                 if (xd[0] != xd[1])
483                         close(xd[1]);
484                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
485                         exit(1);
486                 exit(0);
487         }
488         close(xd[0]);
489         close(fd[1]);
490         fd[1] = xd[1];
491         return side_pid;
494 static int get_pack(int xd[2])
496         int status;
497         pid_t pid, side_pid;
498         int fd[2];
499         const char *argv[20];
500         char keep_arg[256];
501         char hdr_arg[256];
502         const char **av;
503         int do_keep = keep_pack;
505         side_pid = setup_sideband(fd, xd);
507         av = argv;
508         *hdr_arg = 0;
509         if (unpack_limit) {
510                 struct pack_header header;
512                 if (read_pack_header(fd[0], &header))
513                         die("protocol error: bad pack header");
514                 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
515                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
516                 if (ntohl(header.hdr_entries) < unpack_limit)
517                         do_keep = 0;
518                 else
519                         do_keep = 1;
520         }
522         if (do_keep) {
523                 *av++ = "index-pack";
524                 *av++ = "--stdin";
525                 if (!quiet && !no_progress)
526                         *av++ = "-v";
527                 if (use_thin_pack)
528                         *av++ = "--fix-thin";
529                 if (keep_pack > 1 || unpack_limit) {
530                         int s = sprintf(keep_arg,
531                                         "--keep=fetch-pack %d on ", getpid());
532                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
533                                 strcpy(keep_arg + s, "localhost");
534                         *av++ = keep_arg;
535                 }
536         }
537         else {
538                 *av++ = "unpack-objects";
539                 if (quiet)
540                         *av++ = "-q";
541         }
542         if (*hdr_arg)
543                 *av++ = hdr_arg;
544         *av++ = NULL;
546         pid = fork();
547         if (pid < 0)
548                 die("fetch-pack: unable to fork off %s", argv[0]);
549         if (!pid) {
550                 dup2(fd[0], 0);
551                 close(fd[0]);
552                 close(fd[1]);
553                 execv_git_cmd(argv);
554                 die("%s exec failed", argv[0]);
555         }
556         close(fd[0]);
557         close(fd[1]);
558         while (waitpid(pid, &status, 0) < 0) {
559                 if (errno != EINTR)
560                         die("waiting for %s: %s", argv[0], strerror(errno));
561         }
562         if (WIFEXITED(status)) {
563                 int code = WEXITSTATUS(status);
564                 if (code)
565                         die("%s died with error code %d", argv[0], code);
566                 return 0;
567         }
568         if (WIFSIGNALED(status)) {
569                 int sig = WTERMSIG(status);
570                 die("%s died of signal %d", argv[0], sig);
571         }
572         die("%s died of unnatural causes %d", argv[0], status);
575 static int fetch_pack(int fd[2], int nr_match, char **match)
577         struct ref *ref;
578         unsigned char sha1[20];
580         get_remote_heads(fd[0], &ref, 0, NULL, 0);
581         if (is_repository_shallow() && !server_supports("shallow"))
582                 die("Server does not support shallow clients");
583         if (server_supports("multi_ack")) {
584                 if (verbose)
585                         fprintf(stderr, "Server supports multi_ack\n");
586                 multi_ack = 1;
587         }
588         if (server_supports("side-band-64k")) {
589                 if (verbose)
590                         fprintf(stderr, "Server supports side-band-64k\n");
591                 use_sideband = 2;
592         }
593         else if (server_supports("side-band")) {
594                 if (verbose)
595                         fprintf(stderr, "Server supports side-band\n");
596                 use_sideband = 1;
597         }
598         if (!ref) {
599                 packet_flush(fd[1]);
600                 die("no matching remote head");
601         }
602         if (everything_local(&ref, nr_match, match)) {
603                 packet_flush(fd[1]);
604                 goto all_done;
605         }
606         if (find_common(fd, sha1, ref) < 0)
607                 if (keep_pack != 1)
608                         /* When cloning, it is not unusual to have
609                          * no common commit.
610                          */
611                         fprintf(stderr, "warning: no common commits\n");
613         if (get_pack(fd))
614                 die("git-fetch-pack: fetch failed.");
616  all_done:
617         while (ref) {
618                 printf("%s %s\n",
619                        sha1_to_hex(ref->old_sha1), ref->name);
620                 ref = ref->next;
621         }
622         return 0;
625 static int remove_duplicates(int nr_heads, char **heads)
627         int src, dst;
629         for (src = dst = 0; src < nr_heads; src++) {
630                 /* If heads[src] is different from any of
631                  * heads[0..dst], push it in.
632                  */
633                 int i;
634                 for (i = 0; i < dst; i++) {
635                         if (!strcmp(heads[i], heads[src]))
636                                 break;
637                 }
638                 if (i < dst)
639                         continue;
640                 if (src != dst)
641                         heads[dst] = heads[src];
642                 dst++;
643         }
644         heads[dst] = 0;
645         return dst;
648 static int fetch_pack_config(const char *var, const char *value)
650         if (strcmp(var, "fetch.unpacklimit") == 0) {
651                 fetch_unpack_limit = git_config_int(var, value);
652                 return 0;
653         }
655         if (strcmp(var, "transfer.unpacklimit") == 0) {
656                 transfer_unpack_limit = git_config_int(var, value);
657                 return 0;
658         }
660         return git_default_config(var, value);
663 static struct lock_file lock;
665 int main(int argc, char **argv)
667         int i, ret, nr_heads;
668         char *dest = NULL, **heads;
669         int fd[2];
670         pid_t pid;
671         struct stat st;
673         setup_git_directory();
674         git_config(fetch_pack_config);
676         if (0 <= transfer_unpack_limit)
677                 unpack_limit = transfer_unpack_limit;
678         else if (0 <= fetch_unpack_limit)
679                 unpack_limit = fetch_unpack_limit;
681         nr_heads = 0;
682         heads = NULL;
683         for (i = 1; i < argc; i++) {
684                 char *arg = argv[i];
686                 if (*arg == '-') {
687                         if (!strncmp("--upload-pack=", arg, 14)) {
688                                 uploadpack = arg + 14;
689                                 continue;
690                         }
691                         if (!strncmp("--exec=", arg, 7)) {
692                                 uploadpack = arg + 7;
693                                 continue;
694                         }
695                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
696                                 quiet = 1;
697                                 continue;
698                         }
699                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
700                                 keep_pack++;
701                                 unpack_limit = 0;
702                                 continue;
703                         }
704                         if (!strcmp("--thin", arg)) {
705                                 use_thin_pack = 1;
706                                 continue;
707                         }
708                         if (!strcmp("--all", arg)) {
709                                 fetch_all = 1;
710                                 continue;
711                         }
712                         if (!strcmp("-v", arg)) {
713                                 verbose = 1;
714                                 continue;
715                         }
716                         if (!strncmp("--depth=", arg, 8)) {
717                                 depth = strtol(arg + 8, NULL, 0);
718                                 if (stat(git_path("shallow"), &st))
719                                         st.st_mtime = 0;
720                                 continue;
721                         }
722                         if (!strcmp("--no-progress", arg)) {
723                                 no_progress = 1;
724                                 continue;
725                         }
726                         usage(fetch_pack_usage);
727                 }
728                 dest = arg;
729                 heads = argv + i + 1;
730                 nr_heads = argc - i - 1;
731                 break;
732         }
733         if (!dest)
734                 usage(fetch_pack_usage);
735         if (no_progress) {
736                 char buf[256];
737                 snprintf(buf, sizeof(buf), "%s --no-progress", uploadpack);
738                 pid = git_connect(fd, dest, buf);
739         } else
740                 pid = git_connect(fd, dest, uploadpack);
741         if (pid < 0)
742                 return 1;
743         if (heads && nr_heads)
744                 nr_heads = remove_duplicates(nr_heads, heads);
745         ret = fetch_pack(fd, nr_heads, heads);
746         close(fd[0]);
747         close(fd[1]);
748         ret |= finish_connect(pid);
750         if (!ret && nr_heads) {
751                 /* If the heads to pull were given, we should have
752                  * consumed all of them by matching the remote.
753                  * Otherwise, 'git-fetch remote no-such-ref' would
754                  * silently succeed without issuing an error.
755                  */
756                 for (i = 0; i < nr_heads; i++)
757                         if (heads[i] && heads[i][0]) {
758                                 error("no such remote ref %s", heads[i]);
759                                 ret = 1;
760                         }
761         }
763         if (!ret && depth > 0) {
764                 struct cache_time mtime;
765                 char *shallow = git_path("shallow");
766                 int fd;
768                 mtime.sec = st.st_mtime;
769 #ifdef USE_NSEC
770                 mtime.usec = st.st_mtim.usec;
771 #endif
772                 if (stat(shallow, &st)) {
773                         if (mtime.sec)
774                                 die("shallow file was removed during fetch");
775                 } else if (st.st_mtime != mtime.sec
776 #ifdef USE_NSEC
777                                 || st.st_mtim.usec != mtime.usec
778 #endif
779                           )
780                         die("shallow file was changed during fetch");
782                 fd = hold_lock_file_for_update(&lock, shallow, 1);
783                 if (!write_shallow_commits(fd, 0)) {
784                         unlink(shallow);
785                         rollback_lock_file(&lock);
786                 } else {
787                         close(fd);
788                         commit_lock_file(&lock);
789                 }
790         }
792         return !!ret;