Code

Smart fetch over HTTP: client side
[git.git] / remote-curl.c
1 #include "cache.h"
2 #include "remote.h"
3 #include "strbuf.h"
4 #include "walker.h"
5 #include "http.h"
6 #include "exec_cmd.h"
7 #include "run-command.h"
8 #include "pkt-line.h"
9 #include "sideband.h"
11 static struct remote *remote;
12 static const char *url;
13 static struct walker *walker;
15 struct options {
16         int verbosity;
17         unsigned long depth;
18         unsigned progress : 1,
19                 followtags : 1,
20                 dry_run : 1,
21                 thin : 1;
22 };
23 static struct options options;
25 static void init_walker(void)
26 {
27         if (!walker)
28                 walker = get_http_walker(url, remote);
29 }
31 static int set_option(const char *name, const char *value)
32 {
33         if (!strcmp(name, "verbosity")) {
34                 char *end;
35                 int v = strtol(value, &end, 10);
36                 if (value == end || *end)
37                         return -1;
38                 options.verbosity = v;
39                 return 0;
40         }
41         else if (!strcmp(name, "progress")) {
42                 if (!strcmp(value, "true"))
43                         options.progress = 1;
44                 else if (!strcmp(value, "false"))
45                         options.progress = 0;
46                 else
47                         return -1;
48                 return 0;
49         }
50         else if (!strcmp(name, "depth")) {
51                 char *end;
52                 unsigned long v = strtoul(value, &end, 10);
53                 if (value == end || *end)
54                         return -1;
55                 options.depth = v;
56                 return 0;
57         }
58         else if (!strcmp(name, "followtags")) {
59                 if (!strcmp(value, "true"))
60                         options.followtags = 1;
61                 else if (!strcmp(value, "false"))
62                         options.followtags = 0;
63                 else
64                         return -1;
65                 return 0;
66         }
67         else if (!strcmp(name, "dry-run")) {
68                 if (!strcmp(value, "true"))
69                         options.dry_run = 1;
70                 else if (!strcmp(value, "false"))
71                         options.dry_run = 0;
72                 else
73                         return -1;
74                 return 0;
75         }
76         else {
77                 return 1 /* unsupported */;
78         }
79 }
81 struct discovery {
82         const char *service;
83         char *buf_alloc;
84         char *buf;
85         size_t len;
86         unsigned proto_git : 1;
87 };
88 static struct discovery *last_discovery;
90 static void free_discovery(struct discovery *d)
91 {
92         if (d) {
93                 if (d == last_discovery)
94                         last_discovery = NULL;
95                 free(d->buf_alloc);
96                 free(d);
97         }
98 }
100 static struct discovery* discover_refs(const char *service)
102         struct strbuf buffer = STRBUF_INIT;
103         struct discovery *last = last_discovery;
104         char *refs_url;
105         int http_ret, is_http = 0;
107         if (last && !strcmp(service, last->service))
108                 return last;
109         free_discovery(last);
111         strbuf_addf(&buffer, "%s/info/refs", url);
112         if (!prefixcmp(url, "http://") || !prefixcmp(url, "https://")) {
113                 is_http = 1;
114                 if (!strchr(url, '?'))
115                         strbuf_addch(&buffer, '?');
116                 else
117                         strbuf_addch(&buffer, '&');
118                 strbuf_addf(&buffer, "service=%s", service);
119         }
120         refs_url = strbuf_detach(&buffer, NULL);
122         init_walker();
123         http_ret = http_get_strbuf(refs_url, &buffer, HTTP_NO_CACHE);
124         switch (http_ret) {
125         case HTTP_OK:
126                 break;
127         case HTTP_MISSING_TARGET:
128                 die("%s not found: did you run git update-server-info on the"
129                     " server?", refs_url);
130         default:
131                 http_error(refs_url, http_ret);
132                 die("HTTP request failed");
133         }
135         last= xcalloc(1, sizeof(*last_discovery));
136         last->service = service;
137         last->buf_alloc = strbuf_detach(&buffer, &last->len);
138         last->buf = last->buf_alloc;
140         if (is_http && 5 <= last->len && last->buf[4] == '#') {
141                 /* smart HTTP response; validate that the service
142                  * pkt-line matches our request.
143                  */
144                 struct strbuf exp = STRBUF_INIT;
146                 if (packet_get_line(&buffer, &last->buf, &last->len) <= 0)
147                         die("%s has invalid packet header", refs_url);
148                 if (buffer.len && buffer.buf[buffer.len - 1] == '\n')
149                         strbuf_setlen(&buffer, buffer.len - 1);
151                 strbuf_addf(&exp, "# service=%s", service);
152                 if (strbuf_cmp(&exp, &buffer))
153                         die("invalid server response; got '%s'", buffer.buf);
154                 strbuf_release(&exp);
156                 /* The header can include additional metadata lines, up
157                  * until a packet flush marker.  Ignore these now, but
158                  * in the future we might start to scan them.
159                  */
160                 strbuf_reset(&buffer);
161                 while (packet_get_line(&buffer, &last->buf, &last->len) > 0)
162                         strbuf_reset(&buffer);
164                 last->proto_git = 1;
165         }
167         free(refs_url);
168         strbuf_release(&buffer);
169         last_discovery = last;
170         return last;
173 static int write_discovery(int fd, void *data)
175         struct discovery *heads = data;
176         int err = 0;
177         if (write_in_full(fd, heads->buf, heads->len) != heads->len)
178                 err = 1;
179         close(fd);
180         return err;
183 static struct ref *parse_git_refs(struct discovery *heads)
185         struct ref *list = NULL;
186         struct async async;
188         memset(&async, 0, sizeof(async));
189         async.proc = write_discovery;
190         async.data = heads;
192         if (start_async(&async))
193                 die("cannot start thread to parse advertised refs");
194         get_remote_heads(async.out, &list, 0, NULL, 0, NULL);
195         close(async.out);
196         if (finish_async(&async))
197                 die("ref parsing thread failed");
198         return list;
201 static struct ref *parse_info_refs(struct discovery *heads)
203         char *data, *start, *mid;
204         char *ref_name;
205         int i = 0;
207         struct ref *refs = NULL;
208         struct ref *ref = NULL;
209         struct ref *last_ref = NULL;
211         data = heads->buf;
212         start = NULL;
213         mid = data;
214         while (i < heads->len) {
215                 if (!start) {
216                         start = &data[i];
217                 }
218                 if (data[i] == '\t')
219                         mid = &data[i];
220                 if (data[i] == '\n') {
221                         data[i] = 0;
222                         ref_name = mid + 1;
223                         ref = xmalloc(sizeof(struct ref) +
224                                       strlen(ref_name) + 1);
225                         memset(ref, 0, sizeof(struct ref));
226                         strcpy(ref->name, ref_name);
227                         get_sha1_hex(start, ref->old_sha1);
228                         if (!refs)
229                                 refs = ref;
230                         if (last_ref)
231                                 last_ref->next = ref;
232                         last_ref = ref;
233                         start = NULL;
234                 }
235                 i++;
236         }
238         init_walker();
239         ref = alloc_ref("HEAD");
240         if (!walker->fetch_ref(walker, ref) &&
241             !resolve_remote_symref(ref, refs)) {
242                 ref->next = refs;
243                 refs = ref;
244         } else {
245                 free(ref);
246         }
248         return refs;
251 static struct ref *get_refs(int for_push)
253         struct discovery *heads;
255         if (for_push)
256                 heads = discover_refs("git-receive-pack");
257         else
258                 heads = discover_refs("git-upload-pack");
260         if (heads->proto_git)
261                 return parse_git_refs(heads);
262         return parse_info_refs(heads);
265 static void output_refs(struct ref *refs)
267         struct ref *posn;
268         for (posn = refs; posn; posn = posn->next) {
269                 if (posn->symref)
270                         printf("@%s %s\n", posn->symref, posn->name);
271                 else
272                         printf("%s %s\n", sha1_to_hex(posn->old_sha1), posn->name);
273         }
274         printf("\n");
275         fflush(stdout);
276         free_refs(refs);
279 struct rpc_state {
280         const char *service_name;
281         const char **argv;
282         char *service_url;
283         char *hdr_content_type;
284         char *hdr_accept;
285         char *buf;
286         size_t alloc;
287         size_t len;
288         size_t pos;
289         int in;
290         int out;
291         struct strbuf result;
292 };
294 static size_t rpc_out(void *ptr, size_t eltsize,
295                 size_t nmemb, void *buffer_)
297         size_t max = eltsize * nmemb;
298         struct rpc_state *rpc = buffer_;
299         size_t avail = rpc->len - rpc->pos;
301         if (!avail) {
302                 avail = packet_read_line(rpc->out, rpc->buf, rpc->alloc);
303                 if (!avail)
304                         return 0;
305                 rpc->pos = 0;
306                 rpc->len = avail;
307         }
309         if (max < avail);
310                 avail = max;
311         memcpy(ptr, rpc->buf + rpc->pos, avail);
312         rpc->pos += avail;
313         return avail;
316 static size_t rpc_in(const void *ptr, size_t eltsize,
317                 size_t nmemb, void *buffer_)
319         size_t size = eltsize * nmemb;
320         struct rpc_state *rpc = buffer_;
321         write_or_die(rpc->in, ptr, size);
322         return size;
325 static int post_rpc(struct rpc_state *rpc)
327         struct active_request_slot *slot;
328         struct slot_results results;
329         struct curl_slist *headers = NULL;
330         int err = 0, large_request = 0;
332         /* Try to load the entire request, if we can fit it into the
333          * allocated buffer space we can use HTTP/1.0 and avoid the
334          * chunked encoding mess.
335          */
336         while (1) {
337                 size_t left = rpc->alloc - rpc->len;
338                 char *buf = rpc->buf + rpc->len;
339                 int n;
341                 if (left < LARGE_PACKET_MAX) {
342                         large_request = 1;
343                         break;
344                 }
346                 n = packet_read_line(rpc->out, buf, left);
347                 if (!n)
348                         break;
349                 rpc->len += n;
350         }
352         slot = get_active_slot();
353         slot->results = &results;
355         curl_easy_setopt(slot->curl, CURLOPT_POST, 1);
356         curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
357         curl_easy_setopt(slot->curl, CURLOPT_URL, rpc->service_url);
359         headers = curl_slist_append(headers, rpc->hdr_content_type);
360         headers = curl_slist_append(headers, rpc->hdr_accept);
362         if (large_request) {
363                 /* The request body is large and the size cannot be predicted.
364                  * We must use chunked encoding to send it.
365                  */
366                 headers = curl_slist_append(headers, "Expect: 100-continue");
367                 headers = curl_slist_append(headers, "Transfer-Encoding: chunked");
368                 curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, rpc_out);
369                 curl_easy_setopt(slot->curl, CURLOPT_INFILE, rpc);
370                 if (options.verbosity > 1) {
371                         fprintf(stderr, "POST %s (chunked)\n", rpc->service_name);
372                         fflush(stderr);
373                 }
375         } else {
376                 /* We know the complete request size in advance, use the
377                  * more normal Content-Length approach.
378                  */
379                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf);
380                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len);
381                 if (options.verbosity > 1) {
382                         fprintf(stderr, "POST %s (%lu bytes)\n",
383                                 rpc->service_name, (unsigned long)rpc->len);
384                         fflush(stderr);
385                 }
386         }
388         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
389         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, rpc_in);
390         curl_easy_setopt(slot->curl, CURLOPT_FILE, rpc);
392         slot->curl_result = curl_easy_perform(slot->curl);
393         finish_active_slot(slot);
395         if (results.curl_result != CURLE_OK) {
396                 err |= error("RPC failed; result=%d, HTTP code = %ld",
397                         results.curl_result, results.http_code);
398         }
400         curl_slist_free_all(headers);
401         return err;
404 static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
406         const char *svc = rpc->service_name;
407         struct strbuf buf = STRBUF_INIT;
408         struct child_process client;
409         int err = 0;
411         init_walker();
412         memset(&client, 0, sizeof(client));
413         client.in = -1;
414         client.out = -1;
415         client.git_cmd = 1;
416         client.argv = rpc->argv;
417         if (start_command(&client))
418                 exit(1);
419         if (heads)
420                 write_or_die(client.in, heads->buf, heads->len);
422         rpc->alloc = http_post_buffer;
423         rpc->buf = xmalloc(rpc->alloc);
424         rpc->in = client.in;
425         rpc->out = client.out;
426         strbuf_init(&rpc->result, 0);
428         strbuf_addf(&buf, "%s/%s", url, svc);
429         rpc->service_url = strbuf_detach(&buf, NULL);
431         strbuf_addf(&buf, "Content-Type: application/x-%s-request", svc);
432         rpc->hdr_content_type = strbuf_detach(&buf, NULL);
434         strbuf_addf(&buf, "Accept: application/x-%s-response", svc);
435         rpc->hdr_accept = strbuf_detach(&buf, NULL);
437         while (!err) {
438                 int n = packet_read_line(rpc->out, rpc->buf, rpc->alloc);
439                 if (!n)
440                         break;
441                 rpc->pos = 0;
442                 rpc->len = n;
443                 err |= post_rpc(rpc);
444         }
445         strbuf_read(&rpc->result, client.out, 0);
447         close(client.in);
448         close(client.out);
449         client.in = -1;
450         client.out = -1;
452         err |= finish_command(&client);
453         free(rpc->service_url);
454         free(rpc->hdr_content_type);
455         free(rpc->hdr_accept);
456         free(rpc->buf);
457         strbuf_release(&buf);
458         return err;
461 static int fetch_dumb(int nr_heads, struct ref **to_fetch)
463         char **targets = xmalloc(nr_heads * sizeof(char*));
464         int ret, i;
466         if (options.depth)
467                 die("dumb http transport does not support --depth");
468         for (i = 0; i < nr_heads; i++)
469                 targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1));
471         init_walker();
472         walker->get_all = 1;
473         walker->get_tree = 1;
474         walker->get_history = 1;
475         walker->get_verbosely = options.verbosity >= 3;
476         walker->get_recover = 0;
477         ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
479         for (i = 0; i < nr_heads; i++)
480                 free(targets[i]);
481         free(targets);
483         return ret ? error("Fetch failed.") : 0;
486 static int fetch_git(struct discovery *heads,
487         int nr_heads, struct ref **to_fetch)
489         struct rpc_state rpc;
490         char *depth_arg = NULL;
491         const char **argv;
492         int argc = 0, i, err;
494         argv = xmalloc((15 + nr_heads) * sizeof(char*));
495         argv[argc++] = "fetch-pack";
496         argv[argc++] = "--stateless-rpc";
497         argv[argc++] = "--lock-pack";
498         if (options.followtags)
499                 argv[argc++] = "--include-tag";
500         if (options.thin)
501                 argv[argc++] = "--thin";
502         if (options.verbosity >= 3) {
503                 argv[argc++] = "-v";
504                 argv[argc++] = "-v";
505         }
506         if (!options.progress)
507                 argv[argc++] = "--no-progress";
508         if (options.depth) {
509                 struct strbuf buf = STRBUF_INIT;
510                 strbuf_addf(&buf, "--depth=%lu", options.depth);
511                 depth_arg = strbuf_detach(&buf, NULL);
512                 argv[argc++] = depth_arg;
513         }
514         argv[argc++] = url;
515         for (i = 0; i < nr_heads; i++) {
516                 struct ref *ref = to_fetch[i];
517                 if (!ref->name || !*ref->name)
518                         die("cannot fetch by sha1 over smart http");
519                 argv[argc++] = ref->name;
520         }
521         argv[argc++] = NULL;
523         memset(&rpc, 0, sizeof(rpc));
524         rpc.service_name = "git-upload-pack",
525         rpc.argv = argv;
527         err = rpc_service(&rpc, heads);
528         if (rpc.result.len)
529                 safe_write(1, rpc.result.buf, rpc.result.len);
530         strbuf_release(&rpc.result);
531         free(argv);
532         free(depth_arg);
533         return err;
536 static int fetch(int nr_heads, struct ref **to_fetch)
538         struct discovery *d = discover_refs("git-upload-pack");
539         if (d->proto_git)
540                 return fetch_git(d, nr_heads, to_fetch);
541         else
542                 return fetch_dumb(nr_heads, to_fetch);
545 static void parse_fetch(struct strbuf *buf)
547         struct ref **to_fetch = NULL;
548         struct ref *list_head = NULL;
549         struct ref **list = &list_head;
550         int alloc_heads = 0, nr_heads = 0;
552         do {
553                 if (!prefixcmp(buf->buf, "fetch ")) {
554                         char *p = buf->buf + strlen("fetch ");
555                         char *name;
556                         struct ref *ref;
557                         unsigned char old_sha1[20];
559                         if (strlen(p) < 40 || get_sha1_hex(p, old_sha1))
560                                 die("protocol error: expected sha/ref, got %s'", p);
561                         if (p[40] == ' ')
562                                 name = p + 41;
563                         else if (!p[40])
564                                 name = "";
565                         else
566                                 die("protocol error: expected sha/ref, got %s'", p);
568                         ref = alloc_ref(name);
569                         hashcpy(ref->old_sha1, old_sha1);
571                         *list = ref;
572                         list = &ref->next;
574                         ALLOC_GROW(to_fetch, nr_heads + 1, alloc_heads);
575                         to_fetch[nr_heads++] = ref;
576                 }
577                 else
578                         die("http transport does not support %s", buf->buf);
580                 strbuf_reset(buf);
581                 if (strbuf_getline(buf, stdin, '\n') == EOF)
582                         return;
583                 if (!*buf->buf)
584                         break;
585         } while (1);
587         if (fetch(nr_heads, to_fetch))
588                 exit(128); /* error already reported */
589         free_refs(list_head);
590         free(to_fetch);
592         printf("\n");
593         fflush(stdout);
594         strbuf_reset(buf);
597 static int push_dav(int nr_spec, char **specs)
599         const char **argv = xmalloc((10 + nr_spec) * sizeof(char*));
600         int argc = 0, i;
602         argv[argc++] = "http-push";
603         argv[argc++] = "--helper-status";
604         if (options.dry_run)
605                 argv[argc++] = "--dry-run";
606         if (options.verbosity > 1)
607                 argv[argc++] = "--verbose";
608         argv[argc++] = url;
609         for (i = 0; i < nr_spec; i++)
610                 argv[argc++] = specs[i];
611         argv[argc++] = NULL;
613         if (run_command_v_opt(argv, RUN_GIT_CMD))
614                 die("git-%s failed", argv[0]);
615         free(argv);
616         return 0;
619 static int push_git(struct discovery *heads, int nr_spec, char **specs)
621         struct rpc_state rpc;
622         const char **argv;
623         int argc = 0, i, err;
625         argv = xmalloc((10 + nr_spec) * sizeof(char*));
626         argv[argc++] = "send-pack";
627         argv[argc++] = "--stateless-rpc";
628         argv[argc++] = "--helper-status";
629         if (options.thin)
630                 argv[argc++] = "--thin";
631         if (options.dry_run)
632                 argv[argc++] = "--dry-run";
633         if (options.verbosity > 1)
634                 argv[argc++] = "--verbose";
635         argv[argc++] = url;
636         for (i = 0; i < nr_spec; i++)
637                 argv[argc++] = specs[i];
638         argv[argc++] = NULL;
640         memset(&rpc, 0, sizeof(rpc));
641         rpc.service_name = "git-receive-pack",
642         rpc.argv = argv;
644         err = rpc_service(&rpc, heads);
645         if (rpc.result.len)
646                 safe_write(1, rpc.result.buf, rpc.result.len);
647         strbuf_release(&rpc.result);
648         free(argv);
649         return err;
652 static int push(int nr_spec, char **specs)
654         struct discovery *heads = discover_refs("git-receive-pack");
655         int ret;
657         if (heads->proto_git)
658                 ret = push_git(heads, nr_spec, specs);
659         else
660                 ret = push_dav(nr_spec, specs);
661         free_discovery(heads);
662         return ret;
665 static void parse_push(struct strbuf *buf)
667         char **specs = NULL;
668         int alloc_spec = 0, nr_spec = 0, i;
670         do {
671                 if (!prefixcmp(buf->buf, "push ")) {
672                         ALLOC_GROW(specs, nr_spec + 1, alloc_spec);
673                         specs[nr_spec++] = xstrdup(buf->buf + 5);
674                 }
675                 else
676                         die("http transport does not support %s", buf->buf);
678                 strbuf_reset(buf);
679                 if (strbuf_getline(buf, stdin, '\n') == EOF)
680                         return;
681                 if (!*buf->buf)
682                         break;
683         } while (1);
685         if (push(nr_spec, specs))
686                 exit(128); /* error already reported */
687         for (i = 0; i < nr_spec; i++)
688                 free(specs[i]);
689         free(specs);
691         printf("\n");
692         fflush(stdout);
695 int main(int argc, const char **argv)
697         struct strbuf buf = STRBUF_INIT;
699         git_extract_argv0_path(argv[0]);
700         setup_git_directory();
701         if (argc < 2) {
702                 fprintf(stderr, "Remote needed\n");
703                 return 1;
704         }
706         options.verbosity = 1;
707         options.progress = !!isatty(2);
708         options.thin = 1;
710         remote = remote_get(argv[1]);
712         if (argc > 2) {
713                 url = argv[2];
714         } else {
715                 url = remote->url[0];
716         }
718         do {
719                 if (strbuf_getline(&buf, stdin, '\n') == EOF)
720                         break;
721                 if (!prefixcmp(buf.buf, "fetch ")) {
722                         parse_fetch(&buf);
724                 } else if (!strcmp(buf.buf, "list") || !prefixcmp(buf.buf, "list ")) {
725                         int for_push = !!strstr(buf.buf + 4, "for-push");
726                         output_refs(get_refs(for_push));
728                 } else if (!prefixcmp(buf.buf, "push ")) {
729                         parse_push(&buf);
731                 } else if (!prefixcmp(buf.buf, "option ")) {
732                         char *name = buf.buf + strlen("option ");
733                         char *value = strchr(name, ' ');
734                         int result;
736                         if (value)
737                                 *value++ = '\0';
738                         else
739                                 value = "true";
741                         result = set_option(name, value);
742                         if (!result)
743                                 printf("ok\n");
744                         else if (result < 0)
745                                 printf("error invalid value\n");
746                         else
747                                 printf("unsupported\n");
748                         fflush(stdout);
750                 } else if (!strcmp(buf.buf, "capabilities")) {
751                         printf("fetch\n");
752                         printf("option\n");
753                         printf("push\n");
754                         printf("\n");
755                         fflush(stdout);
756                 } else {
757                         return 1;
758                 }
759                 strbuf_reset(&buf);
760         } while (1);
761         return 0;