From: Junio C Hamano Date: Sat, 21 Nov 2009 07:51:23 +0000 (-0800) Subject: Merge branch 'sp/smart-http' X-Git-Tag: v1.6.6-rc0~22 X-Git-Url: https://git.tokkee.org/?a=commitdiff_plain;h=905bf7742cf5f4a6dea2e75ba2dbd89d5dfaa793;p=git.git Merge branch 'sp/smart-http' * sp/smart-http: (37 commits) http-backend: Let gcc check the format of more printf-type functions. http-backend: Fix access beyond end of string. http-backend: Fix bad treatment of uintmax_t in Content-Length t5551-http-fetch: Work around broken Accept header in libcurl t5551-http-fetch: Work around some libcurl versions http-backend: Protect GIT_PROJECT_ROOT from /../ requests Git-aware CGI to provide dumb HTTP transport http-backend: Test configuration options http-backend: Use http.getanyfile to disable dumb HTTP serving test smart http fetch and push http tests: use /dumb/ URL prefix set httpd port before sourcing lib-httpd t5540-http-push: remove redundant fetches Smart HTTP fetch: gzip requests Smart fetch over HTTP: client side Smart push over HTTP: client side Discover refs via smart HTTP server when available http-backend: more explict LocationMatch http-backend: add example for gitweb on same URL http-backend: use mod_alias instead of mod_rewrite ... Conflicts: .gitignore remote-curl.c --- 905bf7742cf5f4a6dea2e75ba2dbd89d5dfaa793 diff --cc .gitignore index 338c6d888,353d22fc9..f63992a6c --- a/.gitignore +++ b/.gitignore @@@ -1,166 -1,167 +1,167 @@@ -GIT-BUILD-OPTIONS -GIT-CFLAGS -GIT-GUI-VARS -GIT-VERSION-FILE -git -git-add -git-add--interactive -git-am -git-annotate -git-apply -git-archimport -git-archive -git-bisect -git-bisect--helper -git-blame -git-branch -git-bundle -git-cat-file -git-check-attr -git-check-ref-format -git-checkout -git-checkout-index -git-cherry -git-cherry-pick -git-clean -git-clone -git-commit -git-commit-tree -git-config -git-count-objects -git-cvsexportcommit -git-cvsimport -git-cvsserver -git-daemon -git-diff -git-diff-files -git-diff-index -git-diff-tree -git-difftool -git-difftool--helper -git-describe -git-fast-export -git-fast-import -git-fetch -git-fetch--tool -git-fetch-pack -git-filter-branch -git-fmt-merge-msg -git-for-each-ref -git-format-patch -git-fsck -git-fsck-objects -git-gc -git-get-tar-commit-id -git-grep -git-hash-object -git-help -git-http-backend -git-http-fetch -git-http-push -git-imap-send -git-index-pack -git-init -git-init-db -git-instaweb -git-log -git-lost-found -git-ls-files -git-ls-remote -git-ls-tree -git-mailinfo -git-mailsplit -git-merge -git-merge-base -git-merge-index -git-merge-file -git-merge-tree -git-merge-octopus -git-merge-one-file -git-merge-ours -git-merge-recursive -git-merge-resolve -git-merge-subtree -git-mergetool -git-mergetool--lib -git-mktag -git-mktree -git-name-rev -git-mv -git-pack-redundant -git-pack-objects -git-pack-refs -git-parse-remote -git-patch-id -git-peek-remote -git-prune -git-prune-packed -git-pull -git-push -git-quiltimport -git-read-tree -git-rebase -git-rebase--interactive -git-receive-pack -git-reflog -git-relink -git-remote -git-remote-curl -git-repack -git-replace -git-repo-config -git-request-pull -git-rerere -git-reset -git-rev-list -git-rev-parse -git-revert -git-rm -git-send-email -git-send-pack -git-sh-setup -git-shell -git-shortlog -git-show -git-show-branch -git-show-index -git-show-ref -git-stage -git-stash -git-status -git-stripspace -git-submodule -git-svn -git-symbolic-ref -git-tag -git-tar-tree -git-unpack-file -git-unpack-objects -git-update-index -git-update-ref -git-update-server-info -git-upload-archive -git-upload-pack -git-var -git-verify-pack -git-verify-tag -git-web--browse -git-whatchanged -git-write-tree -git-core-*/?* -gitk-wish -gitweb/gitweb.cgi -test-chmtime -test-ctype -test-date -test-delta -test-dump-cache-tree -test-genrandom -test-match-trees -test-parse-options -test-path-utils -test-sha1 -test-sigchain -common-cmds.h +/GIT-BUILD-OPTIONS +/GIT-CFLAGS +/GIT-GUI-VARS +/GIT-VERSION-FILE +/git +/git-add +/git-add--interactive +/git-am +/git-annotate +/git-apply +/git-archimport +/git-archive +/git-bisect +/git-bisect--helper +/git-blame +/git-branch +/git-bundle +/git-cat-file +/git-check-attr +/git-check-ref-format +/git-checkout +/git-checkout-index +/git-cherry +/git-cherry-pick +/git-clean +/git-clone +/git-commit +/git-commit-tree +/git-config +/git-count-objects +/git-cvsexportcommit +/git-cvsimport +/git-cvsserver +/git-daemon +/git-diff +/git-diff-files +/git-diff-index +/git-diff-tree +/git-difftool +/git-difftool--helper +/git-describe +/git-fast-export +/git-fast-import +/git-fetch +/git-fetch--tool +/git-fetch-pack +/git-filter-branch +/git-fmt-merge-msg +/git-for-each-ref +/git-format-patch +/git-fsck +/git-fsck-objects +/git-gc +/git-get-tar-commit-id +/git-grep +/git-hash-object +/git-help ++/git-http-backend +/git-http-fetch +/git-http-push +/git-imap-send +/git-index-pack +/git-init +/git-init-db +/git-instaweb +/git-log +/git-lost-found +/git-ls-files +/git-ls-remote +/git-ls-tree +/git-mailinfo +/git-mailsplit +/git-merge +/git-merge-base +/git-merge-index +/git-merge-file +/git-merge-tree +/git-merge-octopus +/git-merge-one-file +/git-merge-ours +/git-merge-recursive +/git-merge-resolve +/git-merge-subtree +/git-mergetool +/git-mergetool--lib +/git-mktag +/git-mktree +/git-name-rev +/git-mv +/git-pack-redundant +/git-pack-objects +/git-pack-refs +/git-parse-remote +/git-patch-id +/git-peek-remote +/git-prune +/git-prune-packed +/git-pull +/git-push +/git-quiltimport +/git-read-tree +/git-rebase +/git-rebase--interactive +/git-receive-pack +/git-reflog +/git-relink +/git-remote +/git-remote-curl +/git-repack +/git-replace +/git-repo-config +/git-request-pull +/git-rerere +/git-reset +/git-rev-list +/git-rev-parse +/git-revert +/git-rm +/git-send-email +/git-send-pack +/git-sh-setup +/git-shell +/git-shortlog +/git-show +/git-show-branch +/git-show-index +/git-show-ref +/git-stage +/git-stash +/git-status +/git-stripspace +/git-submodule +/git-svn +/git-symbolic-ref +/git-tag +/git-tar-tree +/git-unpack-file +/git-unpack-objects +/git-update-index +/git-update-ref +/git-update-server-info +/git-upload-archive +/git-upload-pack +/git-var +/git-verify-pack +/git-verify-tag +/git-web--browse +/git-whatchanged +/git-write-tree +/git-core-*/?* +/gitk-git/gitk-wish +/gitweb/gitweb.cgi +/test-chmtime +/test-ctype +/test-date +/test-delta +/test-dump-cache-tree +/test-genrandom +/test-match-trees +/test-parse-options +/test-path-utils +/test-sha1 +/test-sigchain +/common-cmds.h *.tar.gz *.dsc *.deb diff --cc remote-curl.c index ebdab3603,0d7cf16e9..4f28c222f --- a/remote-curl.c +++ b/remote-curl.c @@@ -74,18 -248,506 +248,507 @@@ static struct ref *parse_info_refs(stru return refs; } + static struct ref *get_refs(int for_push) + { + struct discovery *heads; + + if (for_push) + heads = discover_refs("git-receive-pack"); + else + heads = discover_refs("git-upload-pack"); + + if (heads->proto_git) + return parse_git_refs(heads); + return parse_info_refs(heads); + } + + static void output_refs(struct ref *refs) + { + struct ref *posn; + for (posn = refs; posn; posn = posn->next) { + if (posn->symref) + printf("@%s %s\n", posn->symref, posn->name); + else + printf("%s %s\n", sha1_to_hex(posn->old_sha1), posn->name); + } + printf("\n"); + fflush(stdout); + free_refs(refs); + } + + struct rpc_state { + const char *service_name; + const char **argv; + char *service_url; + char *hdr_content_type; + char *hdr_accept; + char *buf; + size_t alloc; + size_t len; + size_t pos; + int in; + int out; + struct strbuf result; + unsigned gzip_request : 1; + }; + + static size_t rpc_out(void *ptr, size_t eltsize, + size_t nmemb, void *buffer_) + { + size_t max = eltsize * nmemb; + struct rpc_state *rpc = buffer_; + size_t avail = rpc->len - rpc->pos; + + if (!avail) { + avail = packet_read_line(rpc->out, rpc->buf, rpc->alloc); + if (!avail) + return 0; + rpc->pos = 0; + rpc->len = avail; + } + + if (max < avail); + avail = max; + memcpy(ptr, rpc->buf + rpc->pos, avail); + rpc->pos += avail; + return avail; + } + + static size_t rpc_in(const void *ptr, size_t eltsize, + size_t nmemb, void *buffer_) + { + size_t size = eltsize * nmemb; + struct rpc_state *rpc = buffer_; + write_or_die(rpc->in, ptr, size); + return size; + } + + static int post_rpc(struct rpc_state *rpc) + { + struct active_request_slot *slot; + struct slot_results results; + struct curl_slist *headers = NULL; + int use_gzip = rpc->gzip_request; + char *gzip_body = NULL; + int err = 0, large_request = 0; + + /* Try to load the entire request, if we can fit it into the + * allocated buffer space we can use HTTP/1.0 and avoid the + * chunked encoding mess. + */ + while (1) { + size_t left = rpc->alloc - rpc->len; + char *buf = rpc->buf + rpc->len; + int n; + + if (left < LARGE_PACKET_MAX) { + large_request = 1; + use_gzip = 0; + break; + } + + n = packet_read_line(rpc->out, buf, left); + if (!n) + break; + rpc->len += n; + } + + slot = get_active_slot(); + slot->results = &results; + + curl_easy_setopt(slot->curl, CURLOPT_POST, 1); + curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0); + curl_easy_setopt(slot->curl, CURLOPT_URL, rpc->service_url); + curl_easy_setopt(slot->curl, CURLOPT_ENCODING, ""); + + headers = curl_slist_append(headers, rpc->hdr_content_type); + headers = curl_slist_append(headers, rpc->hdr_accept); + + if (large_request) { + /* The request body is large and the size cannot be predicted. + * We must use chunked encoding to send it. + */ + headers = curl_slist_append(headers, "Expect: 100-continue"); + headers = curl_slist_append(headers, "Transfer-Encoding: chunked"); + curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, rpc_out); + curl_easy_setopt(slot->curl, CURLOPT_INFILE, rpc); + if (options.verbosity > 1) { + fprintf(stderr, "POST %s (chunked)\n", rpc->service_name); + fflush(stderr); + } + + } else if (use_gzip && 1024 < rpc->len) { + /* The client backend isn't giving us compressed data so + * we can try to deflate it ourselves, this may save on. + * the transfer time. + */ + size_t size; + z_stream stream; + int ret; + + memset(&stream, 0, sizeof(stream)); + ret = deflateInit2(&stream, Z_BEST_COMPRESSION, + Z_DEFLATED, (15 + 16), + 8, Z_DEFAULT_STRATEGY); + if (ret != Z_OK) + die("cannot deflate request; zlib init error %d", ret); + size = deflateBound(&stream, rpc->len); + gzip_body = xmalloc(size); + + stream.next_in = (unsigned char *)rpc->buf; + stream.avail_in = rpc->len; + stream.next_out = (unsigned char *)gzip_body; + stream.avail_out = size; + + ret = deflate(&stream, Z_FINISH); + if (ret != Z_STREAM_END) + die("cannot deflate request; zlib deflate error %d", ret); + + ret = deflateEnd(&stream); + if (ret != Z_OK) + die("cannot deflate request; zlib end error %d", ret); + + size = stream.total_out; + + headers = curl_slist_append(headers, "Content-Encoding: gzip"); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, size); + + if (options.verbosity > 1) { + fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n", + rpc->service_name, + (unsigned long)rpc->len, (unsigned long)size); + fflush(stderr); + } + } else { + /* We know the complete request size in advance, use the + * more normal Content-Length approach. + */ + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf); + curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, rpc->len); + if (options.verbosity > 1) { + fprintf(stderr, "POST %s (%lu bytes)\n", + rpc->service_name, (unsigned long)rpc->len); + fflush(stderr); + } + } + + curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, rpc_in); + curl_easy_setopt(slot->curl, CURLOPT_FILE, rpc); + + slot->curl_result = curl_easy_perform(slot->curl); + finish_active_slot(slot); + + if (results.curl_result != CURLE_OK) { + err |= error("RPC failed; result=%d, HTTP code = %ld", + results.curl_result, results.http_code); + } + + curl_slist_free_all(headers); + free(gzip_body); + return err; + } + + static int rpc_service(struct rpc_state *rpc, struct discovery *heads) + { + const char *svc = rpc->service_name; + struct strbuf buf = STRBUF_INIT; + struct child_process client; + int err = 0; + + init_walker(); + memset(&client, 0, sizeof(client)); + client.in = -1; + client.out = -1; + client.git_cmd = 1; + client.argv = rpc->argv; + if (start_command(&client)) + exit(1); + if (heads) + write_or_die(client.in, heads->buf, heads->len); + + rpc->alloc = http_post_buffer; + rpc->buf = xmalloc(rpc->alloc); + rpc->in = client.in; + rpc->out = client.out; + strbuf_init(&rpc->result, 0); + + strbuf_addf(&buf, "%s/%s", url, svc); + rpc->service_url = strbuf_detach(&buf, NULL); + + strbuf_addf(&buf, "Content-Type: application/x-%s-request", svc); + rpc->hdr_content_type = strbuf_detach(&buf, NULL); + + strbuf_addf(&buf, "Accept: application/x-%s-response", svc); + rpc->hdr_accept = strbuf_detach(&buf, NULL); + + while (!err) { + int n = packet_read_line(rpc->out, rpc->buf, rpc->alloc); + if (!n) + break; + rpc->pos = 0; + rpc->len = n; + err |= post_rpc(rpc); + } + strbuf_read(&rpc->result, client.out, 0); + + close(client.in); + close(client.out); + client.in = -1; + client.out = -1; + + err |= finish_command(&client); + free(rpc->service_url); + free(rpc->hdr_content_type); + free(rpc->hdr_accept); + free(rpc->buf); + strbuf_release(&buf); + return err; + } + + static int fetch_dumb(int nr_heads, struct ref **to_fetch) + { + char **targets = xmalloc(nr_heads * sizeof(char*)); + int ret, i; + + if (options.depth) + die("dumb http transport does not support --depth"); + for (i = 0; i < nr_heads; i++) + targets[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1)); + + init_walker(); + walker->get_all = 1; + walker->get_tree = 1; + walker->get_history = 1; + walker->get_verbosely = options.verbosity >= 3; + walker->get_recover = 0; + ret = walker_fetch(walker, nr_heads, targets, NULL, NULL); + + for (i = 0; i < nr_heads; i++) + free(targets[i]); + free(targets); + + return ret ? error("Fetch failed.") : 0; + } + + static int fetch_git(struct discovery *heads, + int nr_heads, struct ref **to_fetch) + { + struct rpc_state rpc; + char *depth_arg = NULL; + const char **argv; + int argc = 0, i, err; + + argv = xmalloc((15 + nr_heads) * sizeof(char*)); + argv[argc++] = "fetch-pack"; + argv[argc++] = "--stateless-rpc"; + argv[argc++] = "--lock-pack"; + if (options.followtags) + argv[argc++] = "--include-tag"; + if (options.thin) + argv[argc++] = "--thin"; + if (options.verbosity >= 3) { + argv[argc++] = "-v"; + argv[argc++] = "-v"; + } + if (!options.progress) + argv[argc++] = "--no-progress"; + if (options.depth) { + struct strbuf buf = STRBUF_INIT; + strbuf_addf(&buf, "--depth=%lu", options.depth); + depth_arg = strbuf_detach(&buf, NULL); + argv[argc++] = depth_arg; + } + argv[argc++] = url; + for (i = 0; i < nr_heads; i++) { + struct ref *ref = to_fetch[i]; + if (!ref->name || !*ref->name) + die("cannot fetch by sha1 over smart http"); + argv[argc++] = ref->name; + } + argv[argc++] = NULL; + + memset(&rpc, 0, sizeof(rpc)); + rpc.service_name = "git-upload-pack", + rpc.argv = argv; + rpc.gzip_request = 1; + + err = rpc_service(&rpc, heads); + if (rpc.result.len) + safe_write(1, rpc.result.buf, rpc.result.len); + strbuf_release(&rpc.result); + free(argv); + free(depth_arg); + return err; + } + + static int fetch(int nr_heads, struct ref **to_fetch) + { + struct discovery *d = discover_refs("git-upload-pack"); + if (d->proto_git) + return fetch_git(d, nr_heads, to_fetch); + else + return fetch_dumb(nr_heads, to_fetch); + } + + static void parse_fetch(struct strbuf *buf) + { + struct ref **to_fetch = NULL; + struct ref *list_head = NULL; + struct ref **list = &list_head; + int alloc_heads = 0, nr_heads = 0; + + do { + if (!prefixcmp(buf->buf, "fetch ")) { + char *p = buf->buf + strlen("fetch "); + char *name; + struct ref *ref; + unsigned char old_sha1[20]; + + if (strlen(p) < 40 || get_sha1_hex(p, old_sha1)) + die("protocol error: expected sha/ref, got %s'", p); + if (p[40] == ' ') + name = p + 41; + else if (!p[40]) + name = ""; + else + die("protocol error: expected sha/ref, got %s'", p); + + ref = alloc_ref(name); + hashcpy(ref->old_sha1, old_sha1); + + *list = ref; + list = &ref->next; + + ALLOC_GROW(to_fetch, nr_heads + 1, alloc_heads); + to_fetch[nr_heads++] = ref; + } + else + die("http transport does not support %s", buf->buf); + + strbuf_reset(buf); + if (strbuf_getline(buf, stdin, '\n') == EOF) + return; + if (!*buf->buf) + break; + } while (1); + + if (fetch(nr_heads, to_fetch)) + exit(128); /* error already reported */ + free_refs(list_head); + free(to_fetch); + + printf("\n"); + fflush(stdout); + strbuf_reset(buf); + } + + static int push_dav(int nr_spec, char **specs) + { + const char **argv = xmalloc((10 + nr_spec) * sizeof(char*)); + int argc = 0, i; + + argv[argc++] = "http-push"; + argv[argc++] = "--helper-status"; + if (options.dry_run) + argv[argc++] = "--dry-run"; + if (options.verbosity > 1) + argv[argc++] = "--verbose"; + argv[argc++] = url; + for (i = 0; i < nr_spec; i++) + argv[argc++] = specs[i]; + argv[argc++] = NULL; + + if (run_command_v_opt(argv, RUN_GIT_CMD)) + die("git-%s failed", argv[0]); + free(argv); + return 0; + } + + static int push_git(struct discovery *heads, int nr_spec, char **specs) + { + struct rpc_state rpc; + const char **argv; + int argc = 0, i, err; + + argv = xmalloc((10 + nr_spec) * sizeof(char*)); + argv[argc++] = "send-pack"; + argv[argc++] = "--stateless-rpc"; + argv[argc++] = "--helper-status"; + if (options.thin) + argv[argc++] = "--thin"; + if (options.dry_run) + argv[argc++] = "--dry-run"; + if (options.verbosity > 1) + argv[argc++] = "--verbose"; + argv[argc++] = url; + for (i = 0; i < nr_spec; i++) + argv[argc++] = specs[i]; + argv[argc++] = NULL; + + memset(&rpc, 0, sizeof(rpc)); + rpc.service_name = "git-receive-pack", + rpc.argv = argv; + + err = rpc_service(&rpc, heads); + if (rpc.result.len) + safe_write(1, rpc.result.buf, rpc.result.len); + strbuf_release(&rpc.result); + free(argv); + return err; + } + + static int push(int nr_spec, char **specs) + { + struct discovery *heads = discover_refs("git-receive-pack"); + int ret; + + if (heads->proto_git) + ret = push_git(heads, nr_spec, specs); + else + ret = push_dav(nr_spec, specs); + free_discovery(heads); + return ret; + } + + static void parse_push(struct strbuf *buf) + { + char **specs = NULL; + int alloc_spec = 0, nr_spec = 0, i; + + do { + if (!prefixcmp(buf->buf, "push ")) { + ALLOC_GROW(specs, nr_spec + 1, alloc_spec); + specs[nr_spec++] = xstrdup(buf->buf + 5); + } + else + die("http transport does not support %s", buf->buf); + + strbuf_reset(buf); + if (strbuf_getline(buf, stdin, '\n') == EOF) + return; + if (!*buf->buf) + break; + } while (1); + + if (push(nr_spec, specs)) + exit(128); /* error already reported */ + for (i = 0; i < nr_spec; i++) + free(specs[i]); + free(specs); + + printf("\n"); + fflush(stdout); + } + int main(int argc, const char **argv) { - struct remote *remote; struct strbuf buf = STRBUF_INIT; - const char *url; - struct walker *walker = NULL; + int nongit; git_extract_argv0_path(argv[0]); - setup_git_directory(); + setup_git_directory_gently(&nongit); if (argc < 2) { fprintf(stderr, "Remote needed\n"); return 1; @@@ -103,36 -769,38 +770,40 @@@ if (strbuf_getline(&buf, stdin, '\n') == EOF) break; if (!prefixcmp(buf.buf, "fetch ")) { - char *obj = buf.buf + strlen("fetch "); + if (nongit) + die("Fetch attempted without a local repo"); - if (!walker) - walker = get_http_walker(url, remote); - walker->get_all = 1; - walker->get_tree = 1; - walker->get_history = 1; - walker->get_verbosely = 0; - walker->get_recover = 0; - if (walker_fetch(walker, 1, &obj, NULL, NULL)) - die("Fetch failed."); - printf("\n"); - fflush(stdout); - } else if (!strcmp(buf.buf, "list")) { - struct ref *refs; - struct ref *posn; - if (!walker) - walker = get_http_walker(url, remote); - refs = get_refs(walker, url); - for (posn = refs; posn; posn = posn->next) { - if (posn->symref) - printf("@%s %s\n", posn->symref, posn->name); - else - printf("%s %s\n", sha1_to_hex(posn->old_sha1), posn->name); - } - printf("\n"); + parse_fetch(&buf); + + } else if (!strcmp(buf.buf, "list") || !prefixcmp(buf.buf, "list ")) { + int for_push = !!strstr(buf.buf + 4, "for-push"); + output_refs(get_refs(for_push)); + + } else if (!prefixcmp(buf.buf, "push ")) { + parse_push(&buf); + + } else if (!prefixcmp(buf.buf, "option ")) { + char *name = buf.buf + strlen("option "); + char *value = strchr(name, ' '); + int result; + + if (value) + *value++ = '\0'; + else + value = "true"; + + result = set_option(name, value); + if (!result) + printf("ok\n"); + else if (result < 0) + printf("error invalid value\n"); + else + printf("unsupported\n"); fflush(stdout); + } else if (!strcmp(buf.buf, "capabilities")) { printf("fetch\n"); + printf("option\n"); + printf("push\n"); printf("\n"); fflush(stdout); } else {