Code

Remove unused object-ref code
[git.git] / builtin-fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "run-command.h"
12 static int transfer_unpack_limit = -1;
13 static int fetch_unpack_limit = -1;
14 static int unpack_limit = 100;
15 static struct fetch_pack_args args = {
16         /* .uploadpack = */ "git-upload-pack",
17 };
19 static const char fetch_pack_usage[] =
20 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
22 #define COMPLETE        (1U << 0)
23 #define COMMON          (1U << 1)
24 #define COMMON_REF      (1U << 2)
25 #define SEEN            (1U << 3)
26 #define POPPED          (1U << 4)
28 /*
29  * After sending this many "have"s if we do not get any new ACK , we
30  * give up traversing our history.
31  */
32 #define MAX_IN_VAIN 256
34 static struct commit_list *rev_list;
35 static int non_common_revs, multi_ack, use_sideband;
37 static void rev_list_push(struct commit *commit, int mark)
38 {
39         if (!(commit->object.flags & mark)) {
40                 commit->object.flags |= mark;
42                 if (!(commit->object.parsed))
43                         parse_commit(commit);
45                 insert_by_date(commit, &rev_list);
47                 if (!(commit->object.flags & COMMON))
48                         non_common_revs++;
49         }
50 }
52 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
53 {
54         struct object *o = deref_tag(parse_object(sha1), path, 0);
56         if (o && o->type == OBJ_COMMIT)
57                 rev_list_push((struct commit *)o, SEEN);
59         return 0;
60 }
62 /*
63    This function marks a rev and its ancestors as common.
64    In some cases, it is desirable to mark only the ancestors (for example
65    when only the server does not yet know that they are common).
66 */
68 static void mark_common(struct commit *commit,
69                 int ancestors_only, int dont_parse)
70 {
71         if (commit != NULL && !(commit->object.flags & COMMON)) {
72                 struct object *o = (struct object *)commit;
74                 if (!ancestors_only)
75                         o->flags |= COMMON;
77                 if (!(o->flags & SEEN))
78                         rev_list_push(commit, SEEN);
79                 else {
80                         struct commit_list *parents;
82                         if (!ancestors_only && !(o->flags & POPPED))
83                                 non_common_revs--;
84                         if (!o->parsed && !dont_parse)
85                                 parse_commit(commit);
87                         for (parents = commit->parents;
88                                         parents;
89                                         parents = parents->next)
90                                 mark_common(parents->item, 0, dont_parse);
91                 }
92         }
93 }
95 /*
96   Get the next rev to send, ignoring the common.
97 */
99 static const unsigned char* get_rev(void)
101         struct commit *commit = NULL;
103         while (commit == NULL) {
104                 unsigned int mark;
105                 struct commit_list* parents;
107                 if (rev_list == NULL || non_common_revs == 0)
108                         return NULL;
110                 commit = rev_list->item;
111                 if (!(commit->object.parsed))
112                         parse_commit(commit);
113                 commit->object.flags |= POPPED;
114                 if (!(commit->object.flags & COMMON))
115                         non_common_revs--;
117                 parents = commit->parents;
119                 if (commit->object.flags & COMMON) {
120                         /* do not send "have", and ignore ancestors */
121                         commit = NULL;
122                         mark = COMMON | SEEN;
123                 } else if (commit->object.flags & COMMON_REF)
124                         /* send "have", and ignore ancestors */
125                         mark = COMMON | SEEN;
126                 else
127                         /* send "have", also for its ancestors */
128                         mark = SEEN;
130                 while (parents) {
131                         if (!(parents->item->object.flags & SEEN))
132                                 rev_list_push(parents->item, mark);
133                         if (mark & COMMON)
134                                 mark_common(parents->item, 1, 0);
135                         parents = parents->next;
136                 }
138                 rev_list = rev_list->next;
139         }
141         return commit->object.sha1;
144 static int find_common(int fd[2], unsigned char *result_sha1,
145                        struct ref *refs)
147         int fetching;
148         int count = 0, flushes = 0, retval;
149         const unsigned char *sha1;
150         unsigned in_vain = 0;
151         int got_continue = 0;
153         for_each_ref(rev_list_insert_ref, NULL);
155         fetching = 0;
156         for ( ; refs ; refs = refs->next) {
157                 unsigned char *remote = refs->old_sha1;
158                 struct object *o;
160                 /*
161                  * If that object is complete (i.e. it is an ancestor of a
162                  * local ref), we tell them we have it but do not have to
163                  * tell them about its ancestors, which they already know
164                  * about.
165                  *
166                  * We use lookup_object here because we are only
167                  * interested in the case we *know* the object is
168                  * reachable and we have already scanned it.
169                  */
170                 if (((o = lookup_object(remote)) != NULL) &&
171                                 (o->flags & COMPLETE)) {
172                         continue;
173                 }
175                 if (!fetching)
176                         packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
177                                      sha1_to_hex(remote),
178                                      (multi_ack ? " multi_ack" : ""),
179                                      (use_sideband == 2 ? " side-band-64k" : ""),
180                                      (use_sideband == 1 ? " side-band" : ""),
181                                      (args.use_thin_pack ? " thin-pack" : ""),
182                                      (args.no_progress ? " no-progress" : ""),
183                                      " ofs-delta");
184                 else
185                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
186                 fetching++;
187         }
188         if (is_repository_shallow())
189                 write_shallow_commits(fd[1], 1);
190         if (args.depth > 0)
191                 packet_write(fd[1], "deepen %d", args.depth);
192         packet_flush(fd[1]);
193         if (!fetching)
194                 return 1;
196         if (args.depth > 0) {
197                 char line[1024];
198                 unsigned char sha1[20];
199                 int len;
201                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
202                         if (!prefixcmp(line, "shallow ")) {
203                                 if (get_sha1_hex(line + 8, sha1))
204                                         die("invalid shallow line: %s", line);
205                                 register_shallow(sha1);
206                                 continue;
207                         }
208                         if (!prefixcmp(line, "unshallow ")) {
209                                 if (get_sha1_hex(line + 10, sha1))
210                                         die("invalid unshallow line: %s", line);
211                                 if (!lookup_object(sha1))
212                                         die("object not found: %s", line);
213                                 /* make sure that it is parsed as shallow */
214                                 parse_object(sha1);
215                                 if (unregister_shallow(sha1))
216                                         die("no shallow found: %s", line);
217                                 continue;
218                         }
219                         die("expected shallow/unshallow, got %s", line);
220                 }
221         }
223         flushes = 0;
224         retval = -1;
225         while ((sha1 = get_rev())) {
226                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
227                 if (args.verbose)
228                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
229                 in_vain++;
230                 if (!(31 & ++count)) {
231                         int ack;
233                         packet_flush(fd[1]);
234                         flushes++;
236                         /*
237                          * We keep one window "ahead" of the other side, and
238                          * will wait for an ACK only on the next one
239                          */
240                         if (count == 32)
241                                 continue;
243                         do {
244                                 ack = get_ack(fd[0], result_sha1);
245                                 if (args.verbose && ack)
246                                         fprintf(stderr, "got ack %d %s\n", ack,
247                                                         sha1_to_hex(result_sha1));
248                                 if (ack == 1) {
249                                         flushes = 0;
250                                         multi_ack = 0;
251                                         retval = 0;
252                                         goto done;
253                                 } else if (ack == 2) {
254                                         struct commit *commit =
255                                                 lookup_commit(result_sha1);
256                                         mark_common(commit, 0, 1);
257                                         retval = 0;
258                                         in_vain = 0;
259                                         got_continue = 1;
260                                 }
261                         } while (ack);
262                         flushes--;
263                         if (got_continue && MAX_IN_VAIN < in_vain) {
264                                 if (args.verbose)
265                                         fprintf(stderr, "giving up\n");
266                                 break; /* give up */
267                         }
268                 }
269         }
270 done:
271         packet_write(fd[1], "done\n");
272         if (args.verbose)
273                 fprintf(stderr, "done\n");
274         if (retval != 0) {
275                 multi_ack = 0;
276                 flushes++;
277         }
278         while (flushes || multi_ack) {
279                 int ack = get_ack(fd[0], result_sha1);
280                 if (ack) {
281                         if (args.verbose)
282                                 fprintf(stderr, "got ack (%d) %s\n", ack,
283                                         sha1_to_hex(result_sha1));
284                         if (ack == 1)
285                                 return 0;
286                         multi_ack = 1;
287                         continue;
288                 }
289                 flushes--;
290         }
291         return retval;
294 static struct commit_list *complete;
296 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
298         struct object *o = parse_object(sha1);
300         while (o && o->type == OBJ_TAG) {
301                 struct tag *t = (struct tag *) o;
302                 if (!t->tagged)
303                         break; /* broken repository */
304                 o->flags |= COMPLETE;
305                 o = parse_object(t->tagged->sha1);
306         }
307         if (o && o->type == OBJ_COMMIT) {
308                 struct commit *commit = (struct commit *)o;
309                 commit->object.flags |= COMPLETE;
310                 insert_by_date(commit, &complete);
311         }
312         return 0;
315 static void mark_recent_complete_commits(unsigned long cutoff)
317         while (complete && cutoff <= complete->item->date) {
318                 if (args.verbose)
319                         fprintf(stderr, "Marking %s as complete\n",
320                                 sha1_to_hex(complete->item->object.sha1));
321                 pop_most_recent_commit(&complete, COMPLETE);
322         }
325 static void filter_refs(struct ref **refs, int nr_match, char **match)
327         struct ref **return_refs;
328         struct ref *newlist = NULL;
329         struct ref **newtail = &newlist;
330         struct ref *ref, *next;
331         struct ref *fastarray[32];
333         if (nr_match && !args.fetch_all) {
334                 if (ARRAY_SIZE(fastarray) < nr_match)
335                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
336                 else {
337                         return_refs = fastarray;
338                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
339                 }
340         }
341         else
342                 return_refs = NULL;
344         for (ref = *refs; ref; ref = next) {
345                 next = ref->next;
346                 if (!memcmp(ref->name, "refs/", 5) &&
347                     check_ref_format(ref->name + 5))
348                         ; /* trash */
349                 else if (args.fetch_all &&
350                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
351                         *newtail = ref;
352                         ref->next = NULL;
353                         newtail = &ref->next;
354                         continue;
355                 }
356                 else {
357                         int order = path_match(ref->name, nr_match, match);
358                         if (order) {
359                                 return_refs[order-1] = ref;
360                                 continue; /* we will link it later */
361                         }
362                 }
363                 free(ref);
364         }
366         if (!args.fetch_all) {
367                 int i;
368                 for (i = 0; i < nr_match; i++) {
369                         ref = return_refs[i];
370                         if (ref) {
371                                 *newtail = ref;
372                                 ref->next = NULL;
373                                 newtail = &ref->next;
374                         }
375                 }
376                 if (return_refs != fastarray)
377                         free(return_refs);
378         }
379         *refs = newlist;
382 static int everything_local(struct ref **refs, int nr_match, char **match)
384         struct ref *ref;
385         int retval;
386         unsigned long cutoff = 0;
388         save_commit_buffer = 0;
390         for (ref = *refs; ref; ref = ref->next) {
391                 struct object *o;
393                 o = parse_object(ref->old_sha1);
394                 if (!o)
395                         continue;
397                 /* We already have it -- which may mean that we were
398                  * in sync with the other side at some time after
399                  * that (it is OK if we guess wrong here).
400                  */
401                 if (o->type == OBJ_COMMIT) {
402                         struct commit *commit = (struct commit *)o;
403                         if (!cutoff || cutoff < commit->date)
404                                 cutoff = commit->date;
405                 }
406         }
408         if (!args.depth) {
409                 for_each_ref(mark_complete, NULL);
410                 if (cutoff)
411                         mark_recent_complete_commits(cutoff);
412         }
414         /*
415          * Mark all complete remote refs as common refs.
416          * Don't mark them common yet; the server has to be told so first.
417          */
418         for (ref = *refs; ref; ref = ref->next) {
419                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
420                                              NULL, 0);
422                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
423                         continue;
425                 if (!(o->flags & SEEN)) {
426                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
428                         mark_common((struct commit *)o, 1, 1);
429                 }
430         }
432         filter_refs(refs, nr_match, match);
434         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
435                 const unsigned char *remote = ref->old_sha1;
436                 unsigned char local[20];
437                 struct object *o;
439                 o = lookup_object(remote);
440                 if (!o || !(o->flags & COMPLETE)) {
441                         retval = 0;
442                         if (!args.verbose)
443                                 continue;
444                         fprintf(stderr,
445                                 "want %s (%s)\n", sha1_to_hex(remote),
446                                 ref->name);
447                         continue;
448                 }
450                 hashcpy(ref->new_sha1, local);
451                 if (!args.verbose)
452                         continue;
453                 fprintf(stderr,
454                         "already have %s (%s)\n", sha1_to_hex(remote),
455                         ref->name);
456         }
457         return retval;
460 static int sideband_demux(int fd, void *data)
462         int *xd = data;
464         return recv_sideband("fetch-pack", xd[0], fd, 2);
467 static int get_pack(int xd[2], char **pack_lockfile)
469         struct async demux;
470         const char *argv[20];
471         char keep_arg[256];
472         char hdr_arg[256];
473         const char **av;
474         int do_keep = args.keep_pack;
475         struct child_process cmd;
477         memset(&demux, 0, sizeof(demux));
478         if (use_sideband) {
479                 /* xd[] is talking with upload-pack; subprocess reads from
480                  * xd[0], spits out band#2 to stderr, and feeds us band#1
481                  * through demux->out.
482                  */
483                 demux.proc = sideband_demux;
484                 demux.data = xd;
485                 if (start_async(&demux))
486                         die("fetch-pack: unable to fork off sideband"
487                             " demultiplexer");
488         }
489         else
490                 demux.out = xd[0];
492         memset(&cmd, 0, sizeof(cmd));
493         cmd.argv = argv;
494         av = argv;
495         *hdr_arg = 0;
496         if (!args.keep_pack && unpack_limit) {
497                 struct pack_header header;
499                 if (read_pack_header(demux.out, &header))
500                         die("protocol error: bad pack header");
501                 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
502                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
503                 if (ntohl(header.hdr_entries) < unpack_limit)
504                         do_keep = 0;
505                 else
506                         do_keep = 1;
507         }
509         if (do_keep) {
510                 if (pack_lockfile)
511                         cmd.out = -1;
512                 *av++ = "index-pack";
513                 *av++ = "--stdin";
514                 if (!args.quiet && !args.no_progress)
515                         *av++ = "-v";
516                 if (args.use_thin_pack)
517                         *av++ = "--fix-thin";
518                 if (args.lock_pack || unpack_limit) {
519                         int s = sprintf(keep_arg,
520                                         "--keep=fetch-pack %d on ", getpid());
521                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
522                                 strcpy(keep_arg + s, "localhost");
523                         *av++ = keep_arg;
524                 }
525         }
526         else {
527                 *av++ = "unpack-objects";
528                 if (args.quiet)
529                         *av++ = "-q";
530         }
531         if (*hdr_arg)
532                 *av++ = hdr_arg;
533         *av++ = NULL;
535         cmd.in = demux.out;
536         cmd.git_cmd = 1;
537         if (start_command(&cmd))
538                 die("fetch-pack: unable to fork off %s", argv[0]);
539         if (do_keep && pack_lockfile)
540                 *pack_lockfile = index_pack_lockfile(cmd.out);
542         if (finish_command(&cmd))
543                 die("%s failed", argv[0]);
544         if (use_sideband && finish_async(&demux))
545                 die("error in sideband demultiplexer");
546         return 0;
549 static struct ref *do_fetch_pack(int fd[2],
550                 int nr_match,
551                 char **match,
552                 char **pack_lockfile)
554         struct ref *ref;
555         unsigned char sha1[20];
557         get_remote_heads(fd[0], &ref, 0, NULL, 0);
558         if (is_repository_shallow() && !server_supports("shallow"))
559                 die("Server does not support shallow clients");
560         if (server_supports("multi_ack")) {
561                 if (args.verbose)
562                         fprintf(stderr, "Server supports multi_ack\n");
563                 multi_ack = 1;
564         }
565         if (server_supports("side-band-64k")) {
566                 if (args.verbose)
567                         fprintf(stderr, "Server supports side-band-64k\n");
568                 use_sideband = 2;
569         }
570         else if (server_supports("side-band")) {
571                 if (args.verbose)
572                         fprintf(stderr, "Server supports side-band\n");
573                 use_sideband = 1;
574         }
575         if (!ref) {
576                 packet_flush(fd[1]);
577                 die("no matching remote head");
578         }
579         if (everything_local(&ref, nr_match, match)) {
580                 packet_flush(fd[1]);
581                 goto all_done;
582         }
583         if (find_common(fd, sha1, ref) < 0)
584                 if (!args.keep_pack)
585                         /* When cloning, it is not unusual to have
586                          * no common commit.
587                          */
588                         fprintf(stderr, "warning: no common commits\n");
590         if (get_pack(fd, pack_lockfile))
591                 die("git-fetch-pack: fetch failed.");
593  all_done:
594         return ref;
597 static int remove_duplicates(int nr_heads, char **heads)
599         int src, dst;
601         for (src = dst = 0; src < nr_heads; src++) {
602                 /* If heads[src] is different from any of
603                  * heads[0..dst], push it in.
604                  */
605                 int i;
606                 for (i = 0; i < dst; i++) {
607                         if (!strcmp(heads[i], heads[src]))
608                                 break;
609                 }
610                 if (i < dst)
611                         continue;
612                 if (src != dst)
613                         heads[dst] = heads[src];
614                 dst++;
615         }
616         return dst;
619 static int fetch_pack_config(const char *var, const char *value)
621         if (strcmp(var, "fetch.unpacklimit") == 0) {
622                 fetch_unpack_limit = git_config_int(var, value);
623                 return 0;
624         }
626         if (strcmp(var, "transfer.unpacklimit") == 0) {
627                 transfer_unpack_limit = git_config_int(var, value);
628                 return 0;
629         }
631         return git_default_config(var, value);
634 static struct lock_file lock;
636 static void fetch_pack_setup(void)
638         static int did_setup;
639         if (did_setup)
640                 return;
641         git_config(fetch_pack_config);
642         if (0 <= transfer_unpack_limit)
643                 unpack_limit = transfer_unpack_limit;
644         else if (0 <= fetch_unpack_limit)
645                 unpack_limit = fetch_unpack_limit;
646         did_setup = 1;
649 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
651         int i, ret, nr_heads;
652         struct ref *ref;
653         char *dest = NULL, **heads;
655         nr_heads = 0;
656         heads = NULL;
657         for (i = 1; i < argc; i++) {
658                 const char *arg = argv[i];
660                 if (*arg == '-') {
661                         if (!prefixcmp(arg, "--upload-pack=")) {
662                                 args.uploadpack = arg + 14;
663                                 continue;
664                         }
665                         if (!prefixcmp(arg, "--exec=")) {
666                                 args.uploadpack = arg + 7;
667                                 continue;
668                         }
669                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
670                                 args.quiet = 1;
671                                 continue;
672                         }
673                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
674                                 args.lock_pack = args.keep_pack;
675                                 args.keep_pack = 1;
676                                 continue;
677                         }
678                         if (!strcmp("--thin", arg)) {
679                                 args.use_thin_pack = 1;
680                                 continue;
681                         }
682                         if (!strcmp("--all", arg)) {
683                                 args.fetch_all = 1;
684                                 continue;
685                         }
686                         if (!strcmp("-v", arg)) {
687                                 args.verbose = 1;
688                                 continue;
689                         }
690                         if (!prefixcmp(arg, "--depth=")) {
691                                 args.depth = strtol(arg + 8, NULL, 0);
692                                 continue;
693                         }
694                         if (!strcmp("--no-progress", arg)) {
695                                 args.no_progress = 1;
696                                 continue;
697                         }
698                         usage(fetch_pack_usage);
699                 }
700                 dest = (char *)arg;
701                 heads = (char **)(argv + i + 1);
702                 nr_heads = argc - i - 1;
703                 break;
704         }
705         if (!dest)
706                 usage(fetch_pack_usage);
708         ref = fetch_pack(&args, dest, nr_heads, heads, NULL);
709         ret = !ref;
711         while (ref) {
712                 printf("%s %s\n",
713                        sha1_to_hex(ref->old_sha1), ref->name);
714                 ref = ref->next;
715         }
717         return ret;
720 struct ref *fetch_pack(struct fetch_pack_args *my_args,
721                 const char *dest,
722                 int nr_heads,
723                 char **heads,
724                 char **pack_lockfile)
726         int i, ret;
727         int fd[2];
728         struct child_process *conn;
729         struct ref *ref;
730         struct stat st;
732         fetch_pack_setup();
733         memcpy(&args, my_args, sizeof(args));
734         if (args.depth > 0) {
735                 if (stat(git_path("shallow"), &st))
736                         st.st_mtime = 0;
737         }
739         conn = git_connect(fd, (char *)dest, args.uploadpack,
740                           args.verbose ? CONNECT_VERBOSE : 0);
741         if (heads && nr_heads)
742                 nr_heads = remove_duplicates(nr_heads, heads);
743         ref = do_fetch_pack(fd, nr_heads, heads, pack_lockfile);
744         close(fd[0]);
745         close(fd[1]);
746         ret = finish_connect(conn);
748         if (!ret && nr_heads) {
749                 /* If the heads to pull were given, we should have
750                  * consumed all of them by matching the remote.
751                  * Otherwise, 'git-fetch remote no-such-ref' would
752                  * silently succeed without issuing an error.
753                  */
754                 for (i = 0; i < nr_heads; i++)
755                         if (heads[i] && heads[i][0]) {
756                                 error("no such remote ref %s", heads[i]);
757                                 ret = 1;
758                         }
759         }
761         if (!ret && args.depth > 0) {
762                 struct cache_time mtime;
763                 char *shallow = git_path("shallow");
764                 int fd;
766                 mtime.sec = st.st_mtime;
767 #ifdef USE_NSEC
768                 mtime.usec = st.st_mtim.usec;
769 #endif
770                 if (stat(shallow, &st)) {
771                         if (mtime.sec)
772                                 die("shallow file was removed during fetch");
773                 } else if (st.st_mtime != mtime.sec
774 #ifdef USE_NSEC
775                                 || st.st_mtim.usec != mtime.usec
776 #endif
777                           )
778                         die("shallow file was changed during fetch");
780                 fd = hold_lock_file_for_update(&lock, shallow, 1);
781                 if (!write_shallow_commits(fd, 0)) {
782                         unlink(shallow);
783                         rollback_lock_file(&lock);
784                 } else {
785                         commit_lock_file(&lock);
786                 }
787         }
789         if (ret)
790                 ref = NULL;
792         return ref;