Code

config.c:store_write_pair(): don't read the byte before a malloc'd buffer.
[git.git] / fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
10 static int keep_pack;
11 static int transfer_unpack_limit = -1;
12 static int fetch_unpack_limit = -1;
13 static int unpack_limit = 100;
14 static int quiet;
15 static int verbose;
16 static int fetch_all;
17 static int depth;
18 static int no_progress;
19 static const char fetch_pack_usage[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
21 static const char *uploadpack = "git-upload-pack";
23 #define COMPLETE        (1U << 0)
24 #define COMMON          (1U << 1)
25 #define COMMON_REF      (1U << 2)
26 #define SEEN            (1U << 3)
27 #define POPPED          (1U << 4)
29 /*
30  * After sending this many "have"s if we do not get any new ACK , we
31  * give up traversing our history.
32  */
33 #define MAX_IN_VAIN 256
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
38 static void rev_list_push(struct commit *commit, int mark)
39 {
40         if (!(commit->object.flags & mark)) {
41                 commit->object.flags |= mark;
43                 if (!(commit->object.parsed))
44                         parse_commit(commit);
46                 insert_by_date(commit, &rev_list);
48                 if (!(commit->object.flags & COMMON))
49                         non_common_revs++;
50         }
51 }
53 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
54 {
55         struct object *o = deref_tag(parse_object(sha1), path, 0);
57         if (o && o->type == OBJ_COMMIT)
58                 rev_list_push((struct commit *)o, SEEN);
60         return 0;
61 }
63 /*
64    This function marks a rev and its ancestors as common.
65    In some cases, it is desirable to mark only the ancestors (for example
66    when only the server does not yet know that they are common).
67 */
69 static void mark_common(struct commit *commit,
70                 int ancestors_only, int dont_parse)
71 {
72         if (commit != NULL && !(commit->object.flags & COMMON)) {
73                 struct object *o = (struct object *)commit;
75                 if (!ancestors_only)
76                         o->flags |= COMMON;
78                 if (!(o->flags & SEEN))
79                         rev_list_push(commit, SEEN);
80                 else {
81                         struct commit_list *parents;
83                         if (!ancestors_only && !(o->flags & POPPED))
84                                 non_common_revs--;
85                         if (!o->parsed && !dont_parse)
86                                 parse_commit(commit);
88                         for (parents = commit->parents;
89                                         parents;
90                                         parents = parents->next)
91                                 mark_common(parents->item, 0, dont_parse);
92                 }
93         }
94 }
96 /*
97   Get the next rev to send, ignoring the common.
98 */
100 static const unsigned char* get_rev(void)
102         struct commit *commit = NULL;
104         while (commit == NULL) {
105                 unsigned int mark;
106                 struct commit_list* parents;
108                 if (rev_list == NULL || non_common_revs == 0)
109                         return NULL;
111                 commit = rev_list->item;
112                 if (!(commit->object.parsed))
113                         parse_commit(commit);
114                 commit->object.flags |= POPPED;
115                 if (!(commit->object.flags & COMMON))
116                         non_common_revs--;
118                 parents = commit->parents;
120                 if (commit->object.flags & COMMON) {
121                         /* do not send "have", and ignore ancestors */
122                         commit = NULL;
123                         mark = COMMON | SEEN;
124                 } else if (commit->object.flags & COMMON_REF)
125                         /* send "have", and ignore ancestors */
126                         mark = COMMON | SEEN;
127                 else
128                         /* send "have", also for its ancestors */
129                         mark = SEEN;
131                 while (parents) {
132                         if (!(parents->item->object.flags & SEEN))
133                                 rev_list_push(parents->item, mark);
134                         if (mark & COMMON)
135                                 mark_common(parents->item, 1, 0);
136                         parents = parents->next;
137                 }
139                 rev_list = rev_list->next;
140         }
142         return commit->object.sha1;
145 static int find_common(int fd[2], unsigned char *result_sha1,
146                        struct ref *refs)
148         int fetching;
149         int count = 0, flushes = 0, retval;
150         const unsigned char *sha1;
151         unsigned in_vain = 0;
152         int got_continue = 0;
154         for_each_ref(rev_list_insert_ref, NULL);
156         fetching = 0;
157         for ( ; refs ; refs = refs->next) {
158                 unsigned char *remote = refs->old_sha1;
159                 struct object *o;
161                 /*
162                  * If that object is complete (i.e. it is an ancestor of a
163                  * local ref), we tell them we have it but do not have to
164                  * tell them about its ancestors, which they already know
165                  * about.
166                  *
167                  * We use lookup_object here because we are only
168                  * interested in the case we *know* the object is
169                  * reachable and we have already scanned it.
170                  */
171                 if (((o = lookup_object(remote)) != NULL) &&
172                                 (o->flags & COMPLETE)) {
173                         continue;
174                 }
176                 if (!fetching)
177                         packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
178                                      sha1_to_hex(remote),
179                                      (multi_ack ? " multi_ack" : ""),
180                                      (use_sideband == 2 ? " side-band-64k" : ""),
181                                      (use_sideband == 1 ? " side-band" : ""),
182                                      (use_thin_pack ? " thin-pack" : ""),
183                                      (no_progress ? " no-progress" : ""),
184                                      " ofs-delta");
185                 else
186                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
187                 fetching++;
188         }
189         if (is_repository_shallow())
190                 write_shallow_commits(fd[1], 1);
191         if (depth > 0)
192                 packet_write(fd[1], "deepen %d", depth);
193         packet_flush(fd[1]);
194         if (!fetching)
195                 return 1;
197         if (depth > 0) {
198                 char line[1024];
199                 unsigned char sha1[20];
200                 int len;
202                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
203                         if (!prefixcmp(line, "shallow ")) {
204                                 if (get_sha1_hex(line + 8, sha1))
205                                         die("invalid shallow line: %s", line);
206                                 register_shallow(sha1);
207                                 continue;
208                         }
209                         if (!prefixcmp(line, "unshallow ")) {
210                                 if (get_sha1_hex(line + 10, sha1))
211                                         die("invalid unshallow line: %s", line);
212                                 if (!lookup_object(sha1))
213                                         die("object not found: %s", line);
214                                 /* make sure that it is parsed as shallow */
215                                 parse_object(sha1);
216                                 if (unregister_shallow(sha1))
217                                         die("no shallow found: %s", line);
218                                 continue;
219                         }
220                         die("expected shallow/unshallow, got %s", line);
221                 }
222         }
224         flushes = 0;
225         retval = -1;
226         while ((sha1 = get_rev())) {
227                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
228                 if (verbose)
229                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
230                 in_vain++;
231                 if (!(31 & ++count)) {
232                         int ack;
234                         packet_flush(fd[1]);
235                         flushes++;
237                         /*
238                          * We keep one window "ahead" of the other side, and
239                          * will wait for an ACK only on the next one
240                          */
241                         if (count == 32)
242                                 continue;
244                         do {
245                                 ack = get_ack(fd[0], result_sha1);
246                                 if (verbose && ack)
247                                         fprintf(stderr, "got ack %d %s\n", ack,
248                                                         sha1_to_hex(result_sha1));
249                                 if (ack == 1) {
250                                         flushes = 0;
251                                         multi_ack = 0;
252                                         retval = 0;
253                                         goto done;
254                                 } else if (ack == 2) {
255                                         struct commit *commit =
256                                                 lookup_commit(result_sha1);
257                                         mark_common(commit, 0, 1);
258                                         retval = 0;
259                                         in_vain = 0;
260                                         got_continue = 1;
261                                 }
262                         } while (ack);
263                         flushes--;
264                         if (got_continue && MAX_IN_VAIN < in_vain) {
265                                 if (verbose)
266                                         fprintf(stderr, "giving up\n");
267                                 break; /* give up */
268                         }
269                 }
270         }
271 done:
272         packet_write(fd[1], "done\n");
273         if (verbose)
274                 fprintf(stderr, "done\n");
275         if (retval != 0) {
276                 multi_ack = 0;
277                 flushes++;
278         }
279         while (flushes || multi_ack) {
280                 int ack = get_ack(fd[0], result_sha1);
281                 if (ack) {
282                         if (verbose)
283                                 fprintf(stderr, "got ack (%d) %s\n", ack,
284                                         sha1_to_hex(result_sha1));
285                         if (ack == 1)
286                                 return 0;
287                         multi_ack = 1;
288                         continue;
289                 }
290                 flushes--;
291         }
292         return retval;
295 static struct commit_list *complete;
297 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
299         struct object *o = parse_object(sha1);
301         while (o && o->type == OBJ_TAG) {
302                 struct tag *t = (struct tag *) o;
303                 if (!t->tagged)
304                         break; /* broken repository */
305                 o->flags |= COMPLETE;
306                 o = parse_object(t->tagged->sha1);
307         }
308         if (o && o->type == OBJ_COMMIT) {
309                 struct commit *commit = (struct commit *)o;
310                 commit->object.flags |= COMPLETE;
311                 insert_by_date(commit, &complete);
312         }
313         return 0;
316 static void mark_recent_complete_commits(unsigned long cutoff)
318         while (complete && cutoff <= complete->item->date) {
319                 if (verbose)
320                         fprintf(stderr, "Marking %s as complete\n",
321                                 sha1_to_hex(complete->item->object.sha1));
322                 pop_most_recent_commit(&complete, COMPLETE);
323         }
326 static void filter_refs(struct ref **refs, int nr_match, char **match)
328         struct ref **return_refs;
329         struct ref *newlist = NULL;
330         struct ref **newtail = &newlist;
331         struct ref *ref, *next;
332         struct ref *fastarray[32];
334         if (nr_match && !fetch_all) {
335                 if (ARRAY_SIZE(fastarray) < nr_match)
336                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
337                 else {
338                         return_refs = fastarray;
339                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
340                 }
341         }
342         else
343                 return_refs = NULL;
345         for (ref = *refs; ref; ref = next) {
346                 next = ref->next;
347                 if (!memcmp(ref->name, "refs/", 5) &&
348                     check_ref_format(ref->name + 5))
349                         ; /* trash */
350                 else if (fetch_all &&
351                          (!depth || prefixcmp(ref->name, "refs/tags/") )) {
352                         *newtail = ref;
353                         ref->next = NULL;
354                         newtail = &ref->next;
355                         continue;
356                 }
357                 else {
358                         int order = path_match(ref->name, nr_match, match);
359                         if (order) {
360                                 return_refs[order-1] = ref;
361                                 continue; /* we will link it later */
362                         }
363                 }
364                 free(ref);
365         }
367         if (!fetch_all) {
368                 int i;
369                 for (i = 0; i < nr_match; i++) {
370                         ref = return_refs[i];
371                         if (ref) {
372                                 *newtail = ref;
373                                 ref->next = NULL;
374                                 newtail = &ref->next;
375                         }
376                 }
377                 if (return_refs != fastarray)
378                         free(return_refs);
379         }
380         *refs = newlist;
383 static int everything_local(struct ref **refs, int nr_match, char **match)
385         struct ref *ref;
386         int retval;
387         unsigned long cutoff = 0;
389         track_object_refs = 0;
390         save_commit_buffer = 0;
392         for (ref = *refs; ref; ref = ref->next) {
393                 struct object *o;
395                 o = parse_object(ref->old_sha1);
396                 if (!o)
397                         continue;
399                 /* We already have it -- which may mean that we were
400                  * in sync with the other side at some time after
401                  * that (it is OK if we guess wrong here).
402                  */
403                 if (o->type == OBJ_COMMIT) {
404                         struct commit *commit = (struct commit *)o;
405                         if (!cutoff || cutoff < commit->date)
406                                 cutoff = commit->date;
407                 }
408         }
410         if (!depth) {
411                 for_each_ref(mark_complete, NULL);
412                 if (cutoff)
413                         mark_recent_complete_commits(cutoff);
414         }
416         /*
417          * Mark all complete remote refs as common refs.
418          * Don't mark them common yet; the server has to be told so first.
419          */
420         for (ref = *refs; ref; ref = ref->next) {
421                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
422                                              NULL, 0);
424                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
425                         continue;
427                 if (!(o->flags & SEEN)) {
428                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
430                         mark_common((struct commit *)o, 1, 1);
431                 }
432         }
434         filter_refs(refs, nr_match, match);
436         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
437                 const unsigned char *remote = ref->old_sha1;
438                 unsigned char local[20];
439                 struct object *o;
441                 o = lookup_object(remote);
442                 if (!o || !(o->flags & COMPLETE)) {
443                         retval = 0;
444                         if (!verbose)
445                                 continue;
446                         fprintf(stderr,
447                                 "want %s (%s)\n", sha1_to_hex(remote),
448                                 ref->name);
449                         continue;
450                 }
452                 hashcpy(ref->new_sha1, local);
453                 if (!verbose)
454                         continue;
455                 fprintf(stderr,
456                         "already have %s (%s)\n", sha1_to_hex(remote),
457                         ref->name);
458         }
459         return retval;
462 static pid_t setup_sideband(int fd[2], int xd[2])
464         pid_t side_pid;
466         if (!use_sideband) {
467                 fd[0] = xd[0];
468                 fd[1] = xd[1];
469                 return 0;
470         }
471         /* xd[] is talking with upload-pack; subprocess reads from
472          * xd[0], spits out band#2 to stderr, and feeds us band#1
473          * through our fd[0].
474          */
475         if (pipe(fd) < 0)
476                 die("fetch-pack: unable to set up pipe");
477         side_pid = fork();
478         if (side_pid < 0)
479                 die("fetch-pack: unable to fork off sideband demultiplexer");
480         if (!side_pid) {
481                 /* subprocess */
482                 close(fd[0]);
483                 if (xd[0] != xd[1])
484                         close(xd[1]);
485                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
486                         exit(1);
487                 exit(0);
488         }
489         close(xd[0]);
490         close(fd[1]);
491         fd[1] = xd[1];
492         return side_pid;
495 static int get_pack(int xd[2])
497         int status;
498         pid_t pid, side_pid;
499         int fd[2];
500         const char *argv[20];
501         char keep_arg[256];
502         char hdr_arg[256];
503         const char **av;
504         int do_keep = keep_pack;
506         side_pid = setup_sideband(fd, xd);
508         av = argv;
509         *hdr_arg = 0;
510         if (unpack_limit) {
511                 struct pack_header header;
513                 if (read_pack_header(fd[0], &header))
514                         die("protocol error: bad pack header");
515                 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
516                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
517                 if (ntohl(header.hdr_entries) < unpack_limit)
518                         do_keep = 0;
519                 else
520                         do_keep = 1;
521         }
523         if (do_keep) {
524                 *av++ = "index-pack";
525                 *av++ = "--stdin";
526                 if (!quiet && !no_progress)
527                         *av++ = "-v";
528                 if (use_thin_pack)
529                         *av++ = "--fix-thin";
530                 if (keep_pack > 1 || unpack_limit) {
531                         int s = sprintf(keep_arg,
532                                         "--keep=fetch-pack %d on ", getpid());
533                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
534                                 strcpy(keep_arg + s, "localhost");
535                         *av++ = keep_arg;
536                 }
537         }
538         else {
539                 *av++ = "unpack-objects";
540                 if (quiet)
541                         *av++ = "-q";
542         }
543         if (*hdr_arg)
544                 *av++ = hdr_arg;
545         *av++ = NULL;
547         pid = fork();
548         if (pid < 0)
549                 die("fetch-pack: unable to fork off %s", argv[0]);
550         if (!pid) {
551                 dup2(fd[0], 0);
552                 close(fd[0]);
553                 close(fd[1]);
554                 execv_git_cmd(argv);
555                 die("%s exec failed", argv[0]);
556         }
557         close(fd[0]);
558         close(fd[1]);
559         while (waitpid(pid, &status, 0) < 0) {
560                 if (errno != EINTR)
561                         die("waiting for %s: %s", argv[0], strerror(errno));
562         }
563         if (WIFEXITED(status)) {
564                 int code = WEXITSTATUS(status);
565                 if (code)
566                         die("%s died with error code %d", argv[0], code);
567                 return 0;
568         }
569         if (WIFSIGNALED(status)) {
570                 int sig = WTERMSIG(status);
571                 die("%s died of signal %d", argv[0], sig);
572         }
573         die("%s died of unnatural causes %d", argv[0], status);
576 static int fetch_pack(int fd[2], int nr_match, char **match)
578         struct ref *ref;
579         unsigned char sha1[20];
581         get_remote_heads(fd[0], &ref, 0, NULL, 0);
582         if (is_repository_shallow() && !server_supports("shallow"))
583                 die("Server does not support shallow clients");
584         if (server_supports("multi_ack")) {
585                 if (verbose)
586                         fprintf(stderr, "Server supports multi_ack\n");
587                 multi_ack = 1;
588         }
589         if (server_supports("side-band-64k")) {
590                 if (verbose)
591                         fprintf(stderr, "Server supports side-band-64k\n");
592                 use_sideband = 2;
593         }
594         else if (server_supports("side-band")) {
595                 if (verbose)
596                         fprintf(stderr, "Server supports side-band\n");
597                 use_sideband = 1;
598         }
599         if (!ref) {
600                 packet_flush(fd[1]);
601                 die("no matching remote head");
602         }
603         if (everything_local(&ref, nr_match, match)) {
604                 packet_flush(fd[1]);
605                 goto all_done;
606         }
607         if (find_common(fd, sha1, ref) < 0)
608                 if (keep_pack != 1)
609                         /* When cloning, it is not unusual to have
610                          * no common commit.
611                          */
612                         fprintf(stderr, "warning: no common commits\n");
614         if (get_pack(fd))
615                 die("git-fetch-pack: fetch failed.");
617  all_done:
618         while (ref) {
619                 printf("%s %s\n",
620                        sha1_to_hex(ref->old_sha1), ref->name);
621                 ref = ref->next;
622         }
623         return 0;
626 static int remove_duplicates(int nr_heads, char **heads)
628         int src, dst;
630         for (src = dst = 0; src < nr_heads; src++) {
631                 /* If heads[src] is different from any of
632                  * heads[0..dst], push it in.
633                  */
634                 int i;
635                 for (i = 0; i < dst; i++) {
636                         if (!strcmp(heads[i], heads[src]))
637                                 break;
638                 }
639                 if (i < dst)
640                         continue;
641                 if (src != dst)
642                         heads[dst] = heads[src];
643                 dst++;
644         }
645         heads[dst] = 0;
646         return dst;
649 static int fetch_pack_config(const char *var, const char *value)
651         if (strcmp(var, "fetch.unpacklimit") == 0) {
652                 fetch_unpack_limit = git_config_int(var, value);
653                 return 0;
654         }
656         if (strcmp(var, "transfer.unpacklimit") == 0) {
657                 transfer_unpack_limit = git_config_int(var, value);
658                 return 0;
659         }
661         return git_default_config(var, value);
664 static struct lock_file lock;
666 int main(int argc, char **argv)
668         int i, ret, nr_heads;
669         char *dest = NULL, **heads;
670         int fd[2];
671         pid_t pid;
672         struct stat st;
674         setup_git_directory();
675         git_config(fetch_pack_config);
677         if (0 <= transfer_unpack_limit)
678                 unpack_limit = transfer_unpack_limit;
679         else if (0 <= fetch_unpack_limit)
680                 unpack_limit = fetch_unpack_limit;
682         nr_heads = 0;
683         heads = NULL;
684         for (i = 1; i < argc; i++) {
685                 char *arg = argv[i];
687                 if (*arg == '-') {
688                         if (!prefixcmp(arg, "--upload-pack=")) {
689                                 uploadpack = arg + 14;
690                                 continue;
691                         }
692                         if (!prefixcmp(arg, "--exec=")) {
693                                 uploadpack = arg + 7;
694                                 continue;
695                         }
696                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
697                                 quiet = 1;
698                                 continue;
699                         }
700                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
701                                 keep_pack++;
702                                 unpack_limit = 0;
703                                 continue;
704                         }
705                         if (!strcmp("--thin", arg)) {
706                                 use_thin_pack = 1;
707                                 continue;
708                         }
709                         if (!strcmp("--all", arg)) {
710                                 fetch_all = 1;
711                                 continue;
712                         }
713                         if (!strcmp("-v", arg)) {
714                                 verbose = 1;
715                                 continue;
716                         }
717                         if (!prefixcmp(arg, "--depth=")) {
718                                 depth = strtol(arg + 8, NULL, 0);
719                                 if (stat(git_path("shallow"), &st))
720                                         st.st_mtime = 0;
721                                 continue;
722                         }
723                         if (!strcmp("--no-progress", arg)) {
724                                 no_progress = 1;
725                                 continue;
726                         }
727                         usage(fetch_pack_usage);
728                 }
729                 dest = arg;
730                 heads = argv + i + 1;
731                 nr_heads = argc - i - 1;
732                 break;
733         }
734         if (!dest)
735                 usage(fetch_pack_usage);
736         pid = git_connect(fd, dest, uploadpack, verbose ? CONNECT_VERBOSE : 0);
737         if (pid < 0)
738                 return 1;
739         if (heads && nr_heads)
740                 nr_heads = remove_duplicates(nr_heads, heads);
741         ret = fetch_pack(fd, nr_heads, heads);
742         close(fd[0]);
743         close(fd[1]);
744         ret |= finish_connect(pid);
746         if (!ret && nr_heads) {
747                 /* If the heads to pull were given, we should have
748                  * consumed all of them by matching the remote.
749                  * Otherwise, 'git-fetch remote no-such-ref' would
750                  * silently succeed without issuing an error.
751                  */
752                 for (i = 0; i < nr_heads; i++)
753                         if (heads[i] && heads[i][0]) {
754                                 error("no such remote ref %s", heads[i]);
755                                 ret = 1;
756                         }
757         }
759         if (!ret && depth > 0) {
760                 struct cache_time mtime;
761                 char *shallow = git_path("shallow");
762                 int fd;
764                 mtime.sec = st.st_mtime;
765 #ifdef USE_NSEC
766                 mtime.usec = st.st_mtim.usec;
767 #endif
768                 if (stat(shallow, &st)) {
769                         if (mtime.sec)
770                                 die("shallow file was removed during fetch");
771                 } else if (st.st_mtime != mtime.sec
772 #ifdef USE_NSEC
773                                 || st.st_mtim.usec != mtime.usec
774 #endif
775                           )
776                         die("shallow file was changed during fetch");
778                 fd = hold_lock_file_for_update(&lock, shallow, 1);
779                 if (!write_shallow_commits(fd, 0)) {
780                         unlink(shallow);
781                         rollback_lock_file(&lock);
782                 } else {
783                         close(fd);
784                         commit_lock_file(&lock);
785                 }
786         }
788         return !!ret;