http-fetch: add optional DAV-based pack list

If git is not built with NO_EXPAT, this patch changes git-http-fetch to
attempt using DAV to get a list of remote packs and fall back to using
objects/info/packs if the DAV request fails.

Signed-off-by: Nick Hengeveld <nickh@reactrix.com>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/Makefile b/Makefile
index 145099adaae0a8aeb55a7ea9a0f4b07799e026ec..c7d5ecf4f3bec7dae2eec42de55c1c58fed3efb9 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -513,6 +513,11 @@ exec_cmd.o: exec_cmd.c
 http.o: http.c
        $(CC) -o $*.o -c $(ALL_CFLAGS) -DGIT_USER_AGENT='"git/$(GIT_VERSION)"' $<
 
+ifdef NO_EXPAT
+http-fetch.o: http-fetch.c
+       $(CC) -o $*.o -c $(ALL_CFLAGS) -DNO_EXPAT $<
+endif
+
 git-%$X: %.o $(GITLIBS)
        $(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) $(LIBS)
 
@@ -535,7 +540,7 @@ git-imap-send$X: imap-send.o $(LIB_FILE)
 
 git-http-fetch$X: fetch.o http.o http-fetch.o $(LIB_FILE)
        $(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \
-               $(LIBS) $(CURL_LIBCURL)
+               $(LIBS) $(CURL_LIBCURL) $(EXPAT_LIBEXPAT)
 
 git-http-push$X: revision.o http.o http-push.o $(LIB_FILE)
        $(CC) $(ALL_CFLAGS) -o $@ $(ALL_LDFLAGS) $(filter %.o,$^) \

diff --git a/http-fetch.c b/http-fetch.c
index dc67218ae7caba09add888bc0da165d6c0c10930..71a7dafd696d0da3a9662fb19cd83f87df1ec9b4 100644 (file)
--- a/http-fetch.c
+++ b/http-fetch.c
@@ -4,6 +4,35 @@
 #include "fetch.h"
 #include "http.h"
 
+#ifndef NO_EXPAT
+#include <expat.h>
+
+/* Definitions for DAV requests */
+#define DAV_PROPFIND "PROPFIND"
+#define DAV_PROPFIND_RESP ".multistatus.response"
+#define DAV_PROPFIND_NAME ".multistatus.response.href"
+#define DAV_PROPFIND_COLLECTION ".multistatus.response.propstat.prop.resourcetype.collection"
+#define PROPFIND_ALL_REQUEST "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\n<D:propfind xmlns:D=\"DAV:\">\n<D:allprop/>\n</D:propfind>"
+
+/* Definitions for processing XML DAV responses */
+#ifndef XML_STATUS_OK
+enum XML_Status {
+  XML_STATUS_OK = 1,
+  XML_STATUS_ERROR = 0
+};
+#define XML_STATUS_OK    1
+#define XML_STATUS_ERROR 0
+#endif
+
+/* Flags that control remote_ls processing */
+#define PROCESS_FILES (1u << 0)
+#define PROCESS_DIRS  (1u << 1)
+#define RECURSIVE     (1u << 2)
+
+/* Flags that remote_ls passes to callback functions */
+#define IS_DIR (1u << 0)
+#endif
+
 #define PREV_BUF_SIZE 4096
 #define RANGE_HEADER_SIZE 30
 
@@ -15,6 +44,7 @@ static struct curl_slist *no_pragma_header;
 struct alt_base
 {
        char *base;
+       int path_len;
        int got_indices;
        struct packed_git *packs;
        struct alt_base *next;
@@ -58,6 +88,30 @@ struct alternates_request {
        int http_specific;
 };
 
+#ifndef NO_EXPAT
+struct xml_ctx
+{
+       char *name;
+       int len;
+       char *cdata;
+       void (*userFunc)(struct xml_ctx *ctx, int tag_closed);
+       void *userData;
+};
+
+struct remote_ls_ctx
+{
+       struct alt_base *repo;
+       char *path;
+       void (*userFunc)(struct remote_ls_ctx *ls);
+       void *userData;
+       int flags;
+       char *dentry_name;
+       int dentry_flags;
+       int rc;
+       struct remote_ls_ctx *parent;
+};
+#endif
+
 static struct object_request *object_queue_head = NULL;
 
 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
@@ -500,6 +554,7 @@ static void process_alternates_response(void *callback_data)
                        int serverlen = 0;
                        struct alt_base *newalt;
                        char *target = NULL;
+                       char *path;
                        if (data[i] == '/') {
                                serverlen = strchr(base + 8, '/') - base;
                                okay = 1;
@@ -540,6 +595,13 @@ static void process_alternates_response(void *callback_data)
                                newalt->base = target;
                                newalt->got_indices = 0;
                                newalt->packs = NULL;
+                               path = strstr(target, "//");
+                               if (path) {
+                                       path = index(path+2, '/');
+                                       if (path)
+                                               newalt->path_len = strlen(path);
+                               }
+
                                while (tail->next != NULL)
                                        tail = tail->next;
                                tail->next = newalt;
@@ -611,6 +673,209 @@ static void fetch_alternates(char *base)
        free(url);
 }
 
+#ifndef NO_EXPAT
+static void
+xml_start_tag(void *userData, const char *name, const char **atts)
+{
+       struct xml_ctx *ctx = (struct xml_ctx *)userData;
+       const char *c = index(name, ':');
+       int new_len;
+
+       if (c == NULL)
+               c = name;
+       else
+               c++;
+
+       new_len = strlen(ctx->name) + strlen(c) + 2;
+
+       if (new_len > ctx->len) {
+               ctx->name = xrealloc(ctx->name, new_len);
+               ctx->len = new_len;
+       }
+       strcat(ctx->name, ".");
+       strcat(ctx->name, c);
+
+       if (ctx->cdata) {
+               free(ctx->cdata);
+               ctx->cdata = NULL;
+       }
+
+       ctx->userFunc(ctx, 0);
+}
+
+static void
+xml_end_tag(void *userData, const char *name)
+{
+       struct xml_ctx *ctx = (struct xml_ctx *)userData;
+       const char *c = index(name, ':');
+       char *ep;
+
+       ctx->userFunc(ctx, 1);
+
+       if (c == NULL)
+               c = name;
+       else
+               c++;
+
+       ep = ctx->name + strlen(ctx->name) - strlen(c) - 1;
+       *ep = 0;
+}
+
+static void
+xml_cdata(void *userData, const XML_Char *s, int len)
+{
+       struct xml_ctx *ctx = (struct xml_ctx *)userData;
+       if (ctx->cdata)
+               free(ctx->cdata);
+       ctx->cdata = xcalloc(len+1, 1);
+       strncpy(ctx->cdata, s, len);
+}
+
+static int remote_ls(struct alt_base *repo, const char *path, int flags,
+                    void (*userFunc)(struct remote_ls_ctx *ls),
+                    void *userData);
+
+static void handle_remote_ls_ctx(struct xml_ctx *ctx, int tag_closed)
+{
+       struct remote_ls_ctx *ls = (struct remote_ls_ctx *)ctx->userData;
+
+       if (tag_closed) {
+               if (!strcmp(ctx->name, DAV_PROPFIND_RESP) && ls->dentry_name) {
+                       if (ls->dentry_flags & IS_DIR) {
+                               if (ls->flags & PROCESS_DIRS) {
+                                       ls->userFunc(ls);
+                               }
+                               if (strcmp(ls->dentry_name, ls->path) &&
+                                   ls->flags & RECURSIVE) {
+                                       ls->rc = remote_ls(ls->repo,
+                                                          ls->dentry_name,
+                                                          ls->flags,
+                                                          ls->userFunc,
+                                                          ls->userData);
+                               }
+                       } else if (ls->flags & PROCESS_FILES) {
+                               ls->userFunc(ls);
+                       }
+               } else if (!strcmp(ctx->name, DAV_PROPFIND_NAME) && ctx->cdata) {
+                       ls->dentry_name = xmalloc(strlen(ctx->cdata) -
+                                                 ls->repo->path_len + 1);
+                       strcpy(ls->dentry_name, ctx->cdata + ls->repo->path_len);
+               } else if (!strcmp(ctx->name, DAV_PROPFIND_COLLECTION)) {
+                       ls->dentry_flags |= IS_DIR;
+               }
+       } else if (!strcmp(ctx->name, DAV_PROPFIND_RESP)) {
+               if (ls->dentry_name) {
+                       free(ls->dentry_name);
+               }
+               ls->dentry_name = NULL;
+               ls->dentry_flags = 0;
+       }
+}
+
+static int remote_ls(struct alt_base *repo, const char *path, int flags,
+                    void (*userFunc)(struct remote_ls_ctx *ls),
+                    void *userData)
+{
+       char *url = xmalloc(strlen(repo->base) + strlen(path) + 1);
+       struct active_request_slot *slot;
+       struct slot_results results;
+       struct buffer in_buffer;
+       struct buffer out_buffer;
+       char *in_data;
+       char *out_data;
+       XML_Parser parser = XML_ParserCreate(NULL);
+       enum XML_Status result;
+       struct curl_slist *dav_headers = NULL;
+       struct xml_ctx ctx;
+       struct remote_ls_ctx ls;
+
+       ls.flags = flags;
+       ls.repo = repo;
+       ls.path = strdup(path);
+       ls.dentry_name = NULL;
+       ls.dentry_flags = 0;
+       ls.userData = userData;
+       ls.userFunc = userFunc;
+       ls.rc = 0;
+
+       sprintf(url, "%s%s", repo->base, path);
+
+       out_buffer.size = strlen(PROPFIND_ALL_REQUEST);
+       out_data = xmalloc(out_buffer.size + 1);
+       snprintf(out_data, out_buffer.size + 1, PROPFIND_ALL_REQUEST);
+       out_buffer.posn = 0;
+       out_buffer.buffer = out_data;
+
+       in_buffer.size = 4096;
+       in_data = xmalloc(in_buffer.size);
+       in_buffer.posn = 0;
+       in_buffer.buffer = in_data;
+
+       dav_headers = curl_slist_append(dav_headers, "Depth: 1");
+       dav_headers = curl_slist_append(dav_headers, "Content-Type: text/xml");
+
+       slot = get_active_slot();
+       slot->results = &results;
+       curl_easy_setopt(slot->curl, CURLOPT_INFILE, &out_buffer);
+       curl_easy_setopt(slot->curl, CURLOPT_INFILESIZE, out_buffer.size);
+       curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, fread_buffer);
+       curl_easy_setopt(slot->curl, CURLOPT_FILE, &in_buffer);
+       curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
+       curl_easy_setopt(slot->curl, CURLOPT_URL, url);
+       curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 1);
+       curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, DAV_PROPFIND);
+       curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, dav_headers);
+
+       if (start_active_slot(slot)) {
+               run_active_slot(slot);
+               if (results.curl_result == CURLE_OK) {
+                       ctx.name = xcalloc(10, 1);
+                       ctx.len = 0;
+                       ctx.cdata = NULL;
+                       ctx.userFunc = handle_remote_ls_ctx;
+                       ctx.userData = &ls;
+                       XML_SetUserData(parser, &ctx);
+                       XML_SetElementHandler(parser, xml_start_tag,
+                                             xml_end_tag);
+                       XML_SetCharacterDataHandler(parser, xml_cdata);
+                       result = XML_Parse(parser, in_buffer.buffer,
+                                          in_buffer.posn, 1);
+                       free(ctx.name);
+
+                       if (result != XML_STATUS_OK) {
+                               ls.rc = error("XML error: %s",
+                                             XML_ErrorString(
+                                                     XML_GetErrorCode(parser)));
+                       }
+               } else {
+                       ls.rc = -1;
+               }
+       } else {
+               ls.rc = error("Unable to start PROPFIND request");
+       }
+
+       free(ls.path);
+       free(url);
+       free(out_data);
+       free(in_buffer.buffer);
+       curl_slist_free_all(dav_headers);
+
+       return ls.rc;
+}
+
+static void process_ls_pack(struct remote_ls_ctx *ls)
+{
+       unsigned char sha1[20];
+
+       if (strlen(ls->dentry_name) == 63 &&
+           !strncmp(ls->dentry_name, "objects/pack/pack-", 18) &&
+           !strncmp(ls->dentry_name+58, ".pack", 5)) {
+               get_sha1_hex(ls->dentry_name + 18, sha1);
+               setup_index(ls->repo, sha1);
+       }
+}
+#endif
+
 static int fetch_indices(struct alt_base *repo)
 {
        unsigned char sha1[20];
@@ -633,6 +898,12 @@ static int fetch_indices(struct alt_base *repo)
        if (get_verbosely)
                fprintf(stderr, "Getting pack list for %s\n", repo->base);
 
+#ifndef NO_EXPAT
+       if (remote_ls(repo, "objects/pack/", PROCESS_FILES,
+                     process_ls_pack, NULL) == 0)
+               return 0;
+#endif
+
        url = xmalloc(strlen(repo->base) + 21);
        sprintf(url, "%s/objects/info/packs", repo->base);
 
@@ -947,6 +1218,7 @@ int main(int argc, char **argv)
 {
        char *commit_id;
        char *url;
+       char *path;
        int arg = 1;
        int rc = 0;
 
@@ -987,6 +1259,12 @@ int main(int argc, char **argv)
        alt->got_indices = 0;
        alt->packs = NULL;
        alt->next = NULL;
+       path = strstr(url, "//");
+       if (path) {
+               path = index(path+2, '/');
+               if (path)
+                       alt->path_len = strlen(path);
+       }
 
        if (pull(commit_id))
                rc = 1;