X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=transport.c;h=d44fe7cee7bd8f0e524eff6b516cd2c8b8c919e2;hb=4d8b1dc850bafdf2304a525a768fbfc7aa5361ae;hp=d2cbf3acc117705888a81b27f53f2512e6a22db4;hpb=425b1393139d99d89c7a95906686d9b041f2ee3d;p=git.git diff --git a/transport.c b/transport.c index d2cbf3acc..d44fe7cee 100644 --- a/transport.c +++ b/transport.c @@ -1,15 +1,350 @@ #include "cache.h" #include "transport.h" #include "run-command.h" +#ifndef NO_CURL #include "http.h" +#endif #include "pkt-line.h" #include "fetch-pack.h" #include "walker.h" #include "bundle.h" +#include "dir.h" +#include "refs.h" + +/* rsync support */ + +/* + * We copy packed-refs and refs/ into a temporary file, then read the + * loose refs recursively (sorting whenever possible), and then inserting + * those packed refs that are not yet in the list (not validating, but + * assuming that the file is sorted). + * + * Appears refactoring this from refs.c is too cumbersome. + */ + +static int str_cmp(const void *a, const void *b) +{ + const char *s1 = a; + const char *s2 = b; + + return strcmp(s1, s2); +} + +/* path->buf + name_offset is expected to point to "refs/" */ + +static int read_loose_refs(struct strbuf *path, int name_offset, + struct ref **tail) +{ + DIR *dir = opendir(path->buf); + struct dirent *de; + struct { + char **entries; + int nr, alloc; + } list; + int i, pathlen; + + if (!dir) + return -1; + + memset (&list, 0, sizeof(list)); + + while ((de = readdir(dir))) { + if (de->d_name[0] == '.' && (de->d_name[1] == '\0' || + (de->d_name[1] == '.' && + de->d_name[2] == '\0'))) + continue; + ALLOC_GROW(list.entries, list.nr + 1, list.alloc); + list.entries[list.nr++] = xstrdup(de->d_name); + } + closedir(dir); + + /* sort the list */ + + qsort(list.entries, list.nr, sizeof(char *), str_cmp); + + pathlen = path->len; + strbuf_addch(path, '/'); + + for (i = 0; i < list.nr; i++, strbuf_setlen(path, pathlen + 1)) { + strbuf_addstr(path, list.entries[i]); + if (read_loose_refs(path, name_offset, tail)) { + int fd = open(path->buf, O_RDONLY); + char buffer[40]; + struct ref *next; + + if (fd < 0) + continue; + next = alloc_ref(path->len - name_offset + 1); + if (read_in_full(fd, buffer, 40) != 40 || + get_sha1_hex(buffer, next->old_sha1)) { + close(fd); + free(next); + continue; + } + close(fd); + strcpy(next->name, path->buf + name_offset); + (*tail)->next = next; + *tail = next; + } + } + strbuf_setlen(path, pathlen); + + for (i = 0; i < list.nr; i++) + free(list.entries[i]); + free(list.entries); + + return 0; +} + +/* insert the packed refs for which no loose refs were found */ + +static void insert_packed_refs(const char *packed_refs, struct ref **list) +{ + FILE *f = fopen(packed_refs, "r"); + static char buffer[PATH_MAX]; + + if (!f) + return; + + for (;;) { + int cmp = cmp, len; + + if (!fgets(buffer, sizeof(buffer), f)) { + fclose(f); + return; + } + + if (hexval(buffer[0]) > 0xf) + continue; + len = strlen(buffer); + if (buffer[len - 1] == '\n') + buffer[--len] = '\0'; + if (len < 41) + continue; + while ((*list)->next && + (cmp = strcmp(buffer + 41, + (*list)->next->name)) > 0) + list = &(*list)->next; + if (!(*list)->next || cmp < 0) { + struct ref *next = alloc_ref(len - 40); + buffer[40] = '\0'; + if (get_sha1_hex(buffer, next->old_sha1)) { + warning ("invalid SHA-1: %s", buffer); + free(next); + continue; + } + strcpy(next->name, buffer + 41); + next->next = (*list)->next; + (*list)->next = next; + list = &(*list)->next; + } + } +} + +static struct ref *get_refs_via_rsync(const struct transport *transport) +{ + struct strbuf buf = STRBUF_INIT, temp_dir = STRBUF_INIT; + struct ref dummy, *tail = &dummy; + struct child_process rsync; + const char *args[5]; + int temp_dir_len; + + /* copy the refs to the temporary directory */ + + strbuf_addstr(&temp_dir, git_path("rsync-refs-XXXXXX")); + if (!mkdtemp(temp_dir.buf)) + die ("Could not make temporary directory"); + temp_dir_len = temp_dir.len; + + strbuf_addstr(&buf, transport->url); + strbuf_addstr(&buf, "/refs"); + + memset(&rsync, 0, sizeof(rsync)); + rsync.argv = args; + rsync.stdout_to_stderr = 1; + args[0] = "rsync"; + args[1] = (transport->verbose > 0) ? "-rv" : "-r"; + args[2] = buf.buf; + args[3] = temp_dir.buf; + args[4] = NULL; + + if (run_command(&rsync)) + die ("Could not run rsync to get refs"); + + strbuf_reset(&buf); + strbuf_addstr(&buf, transport->url); + strbuf_addstr(&buf, "/packed-refs"); + + args[2] = buf.buf; + + if (run_command(&rsync)) + die ("Could not run rsync to get refs"); + + /* read the copied refs */ + + strbuf_addstr(&temp_dir, "/refs"); + read_loose_refs(&temp_dir, temp_dir_len + 1, &tail); + strbuf_setlen(&temp_dir, temp_dir_len); + + tail = &dummy; + strbuf_addstr(&temp_dir, "/packed-refs"); + insert_packed_refs(temp_dir.buf, &tail); + strbuf_setlen(&temp_dir, temp_dir_len); + + if (remove_dir_recursively(&temp_dir, 0)) + warning ("Error removing temporary directory %s.", + temp_dir.buf); + + strbuf_release(&buf); + strbuf_release(&temp_dir); + + return dummy.next; +} + +static int fetch_objs_via_rsync(struct transport *transport, + int nr_objs, struct ref **to_fetch) +{ + struct strbuf buf = STRBUF_INIT; + struct child_process rsync; + const char *args[8]; + int result; + + strbuf_addstr(&buf, transport->url); + strbuf_addstr(&buf, "/objects/"); + + memset(&rsync, 0, sizeof(rsync)); + rsync.argv = args; + rsync.stdout_to_stderr = 1; + args[0] = "rsync"; + args[1] = (transport->verbose > 0) ? "-rv" : "-r"; + args[2] = "--ignore-existing"; + args[3] = "--exclude"; + args[4] = "info"; + args[5] = buf.buf; + args[6] = get_object_directory(); + args[7] = NULL; + + /* NEEDSWORK: handle one level of alternates */ + result = run_command(&rsync); + + strbuf_release(&buf); + + return result; +} + +static int write_one_ref(const char *name, const unsigned char *sha1, + int flags, void *data) +{ + struct strbuf *buf = data; + int len = buf->len; + FILE *f; + + /* when called via for_each_ref(), flags is non-zero */ + if (flags && prefixcmp(name, "refs/heads/") && + prefixcmp(name, "refs/tags/")) + return 0; + + strbuf_addstr(buf, name); + if (safe_create_leading_directories(buf->buf) || + !(f = fopen(buf->buf, "w")) || + fprintf(f, "%s\n", sha1_to_hex(sha1)) < 0 || + fclose(f)) + return error("problems writing temporary file %s", buf->buf); + strbuf_setlen(buf, len); + return 0; +} + +static int write_refs_to_temp_dir(struct strbuf *temp_dir, + int refspec_nr, const char **refspec) +{ + int i; + + for (i = 0; i < refspec_nr; i++) { + unsigned char sha1[20]; + char *ref; + + if (dwim_ref(refspec[i], strlen(refspec[i]), sha1, &ref) != 1) + return error("Could not get ref %s", refspec[i]); + + if (write_one_ref(ref, sha1, 0, temp_dir)) { + free(ref); + return -1; + } + free(ref); + } + return 0; +} + +static int rsync_transport_push(struct transport *transport, + int refspec_nr, const char **refspec, int flags) +{ + struct strbuf buf = STRBUF_INIT, temp_dir = STRBUF_INIT; + int result = 0, i; + struct child_process rsync; + const char *args[10]; + + /* first push the objects */ + + strbuf_addstr(&buf, transport->url); + strbuf_addch(&buf, '/'); + + memset(&rsync, 0, sizeof(rsync)); + rsync.argv = args; + rsync.stdout_to_stderr = 1; + i = 0; + args[i++] = "rsync"; + args[i++] = "-a"; + if (flags & TRANSPORT_PUSH_DRY_RUN) + args[i++] = "--dry-run"; + if (transport->verbose > 0) + args[i++] = "-v"; + args[i++] = "--ignore-existing"; + args[i++] = "--exclude"; + args[i++] = "info"; + args[i++] = get_object_directory(); + args[i++] = buf.buf; + args[i++] = NULL; + + if (run_command(&rsync)) + return error("Could not push objects to %s", transport->url); + + /* copy the refs to the temporary directory; they could be packed. */ + + strbuf_addstr(&temp_dir, git_path("rsync-refs-XXXXXX")); + if (!mkdtemp(temp_dir.buf)) + die ("Could not make temporary directory"); + strbuf_addch(&temp_dir, '/'); + + if (flags & TRANSPORT_PUSH_ALL) { + if (for_each_ref(write_one_ref, &temp_dir)) + return -1; + } else if (write_refs_to_temp_dir(&temp_dir, refspec_nr, refspec)) + return -1; + + i = 2; + if (flags & TRANSPORT_PUSH_DRY_RUN) + args[i++] = "--dry-run"; + if (!(flags & TRANSPORT_PUSH_FORCE)) + args[i++] = "--ignore-existing"; + args[i++] = temp_dir.buf; + args[i++] = transport->url; + args[i++] = NULL; + if (run_command(&rsync)) + result = error("Could not push to %s", transport->url); + + if (remove_dir_recursively(&temp_dir, 0)) + warning ("Could not remove temporary directory %s.", + temp_dir.buf); + + strbuf_release(&buf); + strbuf_release(&temp_dir); + + return result; +} /* Generic functions for using commit walkers */ -static int fetch_objs_via_walker(const struct transport *transport, +static int fetch_objs_via_walker(struct transport *transport, int nr_objs, struct ref **to_fetch) { char *dest = xstrdup(transport->url); @@ -20,13 +355,13 @@ static int fetch_objs_via_walker(const struct transport *transport, walker->get_all = 1; walker->get_tree = 1; walker->get_history = 1; - walker->get_verbosely = transport->verbose; + walker->get_verbosely = transport->verbose >= 0; walker->get_recover = 0; for (i = 0; i < nr_objs; i++) objs[i] = xstrdup(sha1_to_hex(to_fetch[i]->old_sha1)); - if (walker_fetch(walker, nr_objs, objs, NULL, dest)) + if (walker_fetch(walker, nr_objs, objs, NULL, NULL)) die("Fetch failed."); for (i = 0; i < nr_objs; i++) @@ -44,8 +379,7 @@ static int disconnect_walker(struct transport *transport) return 0; } -static const struct transport_ops rsync_transport; - +#ifndef NO_CURL static int curl_transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) { const char **argv; int argc; @@ -58,6 +392,8 @@ static int curl_transport_push(struct transport *transport, int refspec_nr, cons argv[argc++] = "--all"; if (flags & TRANSPORT_PUSH_FORCE) argv[argc++] = "--force"; + if (flags & TRANSPORT_PUSH_DRY_RUN) + argv[argc++] = "--dry-run"; argv[argc++] = transport->url; while (refspec_nr--) argv[argc++] = *refspec++; @@ -78,7 +414,6 @@ static int curl_transport_push(struct transport *transport, int refspec_nr, cons return !!err; } -#ifndef NO_CURL static int missing__target(int code, int result) { return /* file:// URL -- do we ever use one??? */ @@ -174,24 +509,16 @@ static struct ref *get_refs_via_curl(const struct transport *transport) return refs; } -#else - -static struct ref *get_refs_via_curl(const struct transport *transport) +static int fetch_objs_via_curl(struct transport *transport, + int nr_objs, struct ref **to_fetch) { - die("Cannot fetch from '%s' without curl ...", transport->url); - return NULL; + if (!transport->data) + transport->data = get_http_walker(transport->url); + return fetch_objs_via_walker(transport, nr_objs, to_fetch); } #endif -static const struct transport_ops curl_transport = { - /* set_option */ NULL, - /* get_refs_list */ get_refs_via_curl, - /* fetch */ fetch_objs_via_walker, - /* push */ curl_transport_push, - /* disconnect */ disconnect_walker -}; - struct bundle_transport_data { int fd; struct bundle_header header; @@ -210,7 +537,7 @@ static struct ref *get_refs_from_bundle(const struct transport *transport) die ("Could not read bundle '%s'.", transport->url); for (i = 0; i < data->header.references.nr; i++) { struct ref_list_entry *e = data->header.references.list + i; - struct ref *ref = alloc_ref(strlen(e->name)); + struct ref *ref = alloc_ref(strlen(e->name) + 1); hashcpy(ref->old_sha1, e->sha1); strcpy(ref->name, e->name); ref->next = result; @@ -219,7 +546,7 @@ static struct ref *get_refs_from_bundle(const struct transport *transport) return result; } -static int fetch_refs_from_bundle(const struct transport *transport, +static int fetch_refs_from_bundle(struct transport *transport, int nr_heads, struct ref **to_fetch) { struct bundle_transport_data *data = transport->data; @@ -231,25 +558,14 @@ static int close_bundle(struct transport *transport) struct bundle_transport_data *data = transport->data; if (data->fd > 0) close(data->fd); + free(data); return 0; } -static const struct transport_ops bundle_transport = { - /* set_option */ NULL, - /* get_refs_list */ get_refs_from_bundle, - /* fetch */ fetch_refs_from_bundle, - /* push */ NULL, - /* disconnect */ close_bundle -}; - struct git_transport_data { unsigned thin : 1; unsigned keep : 1; - - int unpacklimit; - int depth; - const char *uploadpack; const char *receivepack; }; @@ -270,9 +586,6 @@ static int set_git_option(struct transport *connection, } else if (!strcmp(name, TRANS_OPT_KEEP)) { data->keep = !!value; return 0; - } else if (!strcmp(name, TRANS_OPT_UNPACKLIMIT)) { - data->unpacklimit = atoi(value); - return 0; } else if (!strcmp(name, TRANS_OPT_DEPTH)) { if (!value) data->depth = 0; @@ -288,52 +601,46 @@ static struct ref *get_refs_via_connect(const struct transport *transport) struct git_transport_data *data = transport->data; struct ref *refs; int fd[2]; - pid_t pid; char *dest = xstrdup(transport->url); - - pid = git_connect(fd, dest, data->uploadpack, 0); - - if (pid < 0) - die("Failed to connect to \"%s\"", transport->url); + struct child_process *conn = git_connect(fd, dest, data->uploadpack, 0); get_remote_heads(fd[0], &refs, 0, NULL, 0); packet_flush(fd[1]); - finish_connect(pid); + finish_connect(conn); free(dest); return refs; } -static int fetch_refs_via_pack(const struct transport *transport, +static int fetch_refs_via_pack(struct transport *transport, int nr_heads, struct ref **to_fetch) { struct git_transport_data *data = transport->data; char **heads = xmalloc(nr_heads * sizeof(*heads)); + char **origh = xmalloc(nr_heads * sizeof(*origh)); struct ref *refs; char *dest = xstrdup(transport->url); struct fetch_pack_args args; int i; + memset(&args, 0, sizeof(args)); args.uploadpack = data->uploadpack; - args.quiet = 0; args.keep_pack = data->keep; - args.unpacklimit = data->unpacklimit; + args.lock_pack = 1; args.use_thin_pack = data->thin; - args.fetch_all = 0; - args.verbose = transport->verbose; + args.verbose = transport->verbose > 0; args.depth = data->depth; - args.no_progress = 0; - - setup_fetch_pack(&args); for (i = 0; i < nr_heads; i++) - heads[i] = xstrdup(to_fetch[i]->name); - refs = fetch_pack(dest, nr_heads, heads); + origh[i] = heads[i] = xstrdup(to_fetch[i]->name); + refs = fetch_pack(&args, dest, nr_heads, heads, &transport->pack_lockfile); for (i = 0; i < nr_heads; i++) - free(heads[i]); + free(origh[i]); + free(origh); + free(heads); free_refs(refs); free(dest); return 0; @@ -353,6 +660,8 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const argv[argc++] = "--all"; if (flags & TRANSPORT_PUSH_FORCE) argv[argc++] = "--force"; + if (flags & TRANSPORT_PUSH_DRY_RUN) + argv[argc++] = "--dry-run"; if (data->receivepack) { char *rp = xmalloc(strlen(data->receivepack) + 16); sprintf(rp, "--receive-pack=%s", data->receivepack); @@ -383,12 +692,11 @@ static int git_transport_push(struct transport *transport, int refspec_nr, const return !!err; } -static const struct transport_ops git_transport = { - /* set_option */ set_git_option, - /* get_refs_list */ get_refs_via_connect, - /* fetch */ fetch_refs_via_pack, - /* push */ git_transport_push -}; +static int disconnect_git(struct transport *transport) +{ + free(transport->data); + return 0; +} static int is_local(const char *url) { @@ -405,31 +713,46 @@ static int is_file(const char *url) return S_ISREG(buf.st_mode); } -struct transport *transport_get(struct remote *remote, const char *url, - int fetch) +struct transport *transport_get(struct remote *remote, const char *url) { - struct transport *ret = NULL; + struct transport *ret = xcalloc(1, sizeof(*ret)); + + ret->remote = remote; + ret->url = url; + if (!prefixcmp(url, "rsync://")) { - ret = xmalloc(sizeof(*ret)); - ret->data = NULL; - ret->ops = &rsync_transport; - } else if (!prefixcmp(url, "http://") || !prefixcmp(url, "https://") || - !prefixcmp(url, "ftp://")) { - ret = xmalloc(sizeof(*ret)); - ret->ops = &curl_transport; - if (fetch) - ret->data = get_http_walker(url); - else - ret->data = NULL; + ret->get_refs_list = get_refs_via_rsync; + ret->fetch = fetch_objs_via_rsync; + ret->push = rsync_transport_push; + + } else if (!prefixcmp(url, "http://") + || !prefixcmp(url, "https://") + || !prefixcmp(url, "ftp://")) { +#ifdef NO_CURL + error("git was compiled without libcurl support."); +#else + ret->get_refs_list = get_refs_via_curl; + ret->fetch = fetch_objs_via_curl; + ret->push = curl_transport_push; +#endif + ret->disconnect = disconnect_walker; + } else if (is_local(url) && is_file(url)) { struct bundle_transport_data *data = xcalloc(1, sizeof(*data)); - ret = xmalloc(sizeof(*ret)); ret->data = data; - ret->ops = &bundle_transport; + ret->get_refs_list = get_refs_from_bundle; + ret->fetch = fetch_refs_from_bundle; + ret->disconnect = close_bundle; + } else { struct git_transport_data *data = xcalloc(1, sizeof(*data)); - ret = xcalloc(1, sizeof(*ret)); ret->data = data; + ret->set_option = set_git_option; + ret->get_refs_list = get_refs_via_connect; + ret->fetch = fetch_refs_via_pack; + ret->push = git_transport_push; + ret->disconnect = disconnect_git; + data->thin = 1; data->uploadpack = "git-upload-pack"; if (remote && remote->uploadpack) @@ -437,46 +760,31 @@ struct transport *transport_get(struct remote *remote, const char *url, data->receivepack = "git-receive-pack"; if (remote && remote->receivepack) data->receivepack = remote->receivepack; - data->unpacklimit = -1; - ret->ops = &git_transport; - } - if (ret) { - ret->remote = remote; - ret->url = url; - ret->remote_refs = NULL; - ret->fetch = !!fetch; } + return ret; } int transport_set_option(struct transport *transport, const char *name, const char *value) { - int ret = 1; - if (transport->ops->set_option) - ret = transport->ops->set_option(transport, name, value); - if (ret < 0) - fprintf(stderr, "For '%s' option %s cannot be set to '%s'\n", - transport->url, name, value); - if (ret > 0) - fprintf(stderr, "For '%s' option %s is ignored\n", - transport->url, name); - return ret; + if (transport->set_option) + return transport->set_option(transport, name, value); + return 1; } int transport_push(struct transport *transport, int refspec_nr, const char **refspec, int flags) { - if (!transport->ops->push) + if (!transport->push) return 1; - return transport->ops->push(transport, refspec_nr, refspec, flags); + return transport->push(transport, refspec_nr, refspec, flags); } struct ref *transport_get_remote_refs(struct transport *transport) { if (!transport->remote_refs) - transport->remote_refs = - transport->ops->get_refs_list(transport); + transport->remote_refs = transport->get_refs_list(transport); return transport->remote_refs; } @@ -495,16 +803,25 @@ int transport_fetch_refs(struct transport *transport, struct ref *refs) heads[nr_heads++] = rm; } - rc = transport->ops->fetch(transport, nr_heads, heads); + rc = transport->fetch(transport, nr_heads, heads); free(heads); return rc; } +void transport_unlock_pack(struct transport *transport) +{ + if (transport->pack_lockfile) { + unlink(transport->pack_lockfile); + free(transport->pack_lockfile); + transport->pack_lockfile = NULL; + } +} + int transport_disconnect(struct transport *transport) { int ret = 0; - if (transport->ops->disconnect) - ret = transport->ops->disconnect(transport); + if (transport->disconnect) + ret = transport->disconnect(transport); free(transport); return ret; }