Merge branch 'jc/bigfile'

* jc/bigfile:
  Bigfile: teach "git add" to send a large file straight to a pack
  index_fd(): split into two helper functions
  index_fd(): turn write_object and format_check arguments into one flag

diff --combined cache.h
index 28a921dffc144bab15366bbfb0c52ed34a320bed,bc380139590700e88d436e8577d3b433aa4e5d7b..009b365370f0815e4493f4775f96ebda3cfa6e80
--- 1/cache.h
--- 2/cache.h
+++ b/cache.h
@@@ -511,15 -511,18 +511,18 @@@ struct pathspec 
        struct pathspec_item {
                const char *match;
                int len;
 -              unsigned int has_wildcard:1;
 +              unsigned int use_wildcard:1;
        } *items;
  };
  
  extern int init_pathspec(struct pathspec *, const char **);
  extern void free_pathspec(struct pathspec *);
  extern int ce_path_match(const struct cache_entry *ce, const struct pathspec *pathspec);
- extern int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, enum object_type type, const char *path, int format_check);
- extern int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object);
+ 
+ #define HASH_WRITE_OBJECT 1
+ #define HASH_FORMAT_CHECK 2
+ extern int index_fd(unsigned char *sha1, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags);
+ extern int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags);
  extern void fill_stat_cache_info(struct cache_entry *ce, struct stat *st);
  
  #define REFRESH_REALLY                0x0001  /* ignore_valid */
@@@ -606,7 -609,7 +609,7 @@@ enum eol 
  #endif
  };
  
 -extern enum eol eol;
 +extern enum eol core_eol;
  
  enum branch_track {
        BRANCH_TRACK_UNSPECIFIED = -1,
@@@ -676,24 -679,14 +679,24 @@@ extern char *sha1_pack_name(const unsig
  extern char *sha1_pack_index_name(const unsigned char *sha1);
  extern const char *find_unique_abbrev(const unsigned char *sha1, int);
  extern const unsigned char null_sha1[20];
 -static inline int is_null_sha1(const unsigned char *sha1)
 +
 +static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2)
  {
 -      return !memcmp(sha1, null_sha1, 20);
 +      int i;
 +
 +      for (i = 0; i < 20; i++, sha1++, sha2++) {
 +              if (*sha1 != *sha2)
 +                      return *sha1 - *sha2;
 +      }
 +
 +      return 0;
  }
 -static inline int hashcmp(const unsigned char *sha1, const unsigned char *sha2)
 +
 +static inline int is_null_sha1(const unsigned char *sha1)
  {
 -      return memcmp(sha1, sha2, 20);
 +      return !hashcmp(sha1, null_sha1);
  }
 +
  static inline void hashcpy(unsigned char *sha_dst, const unsigned char *sha_src)
  {
        memcpy(sha_dst, sha_src, 20);
@@@ -756,23 -749,13 +759,23 @@@ char *strip_path_suffix(const char *pat
  int daemon_avoid_alias(const char *path);
  int offset_1st_component(const char *path);
  
 -/* Read and unpack a sha1 file into memory, write memory to a sha1 file */
 -extern int sha1_object_info(const unsigned char *, unsigned long *);
 -extern void *read_sha1_file_repl(const unsigned char *sha1, enum object_type *type, unsigned long *size, const unsigned char **replacement);
 +/* object replacement */
 +#define READ_SHA1_FILE_REPLACE 1
 +extern void *read_sha1_file_extended(const unsigned char *sha1, enum object_type *type, unsigned long *size, unsigned flag);
  static inline void *read_sha1_file(const unsigned char *sha1, enum object_type *type, unsigned long *size)
  {
 -      return read_sha1_file_repl(sha1, type, size, NULL);
 +      return read_sha1_file_extended(sha1, type, size, READ_SHA1_FILE_REPLACE);
 +}
 +extern const unsigned char *do_lookup_replace_object(const unsigned char *sha1);
 +static inline const unsigned char *lookup_replace_object(const unsigned char *sha1)
 +{
 +      if (!read_replace_refs)
 +              return sha1;
 +      return do_lookup_replace_object(sha1);
  }
 +
 +/* Read and unpack a sha1 file into memory, write memory to a sha1 file */
 +extern int sha1_object_info(const unsigned char *, unsigned long *);
  extern int hash_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *sha1);
  extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1);
  extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *);
@@@ -810,15 -793,15 +813,15 @@@ struct object_context 
  };
  
  extern int get_sha1(const char *str, unsigned char *sha1);
 -extern int get_sha1_with_mode_1(const char *str, unsigned char *sha1, unsigned *mode, int gently, const char *prefix);
 +extern int get_sha1_with_mode_1(const char *str, unsigned char *sha1, unsigned *mode, int only_to_die, const char *prefix);
  static inline int get_sha1_with_mode(const char *str, unsigned char *sha1, unsigned *mode)
  {
 -      return get_sha1_with_mode_1(str, sha1, mode, 1, NULL);
 +      return get_sha1_with_mode_1(str, sha1, mode, 0, NULL);
  }
 -extern int get_sha1_with_context_1(const char *name, unsigned char *sha1, struct object_context *orc, int gently, const char *prefix);
 +extern int get_sha1_with_context_1(const char *name, unsigned char *sha1, struct object_context *orc, int only_to_die, const char *prefix);
  static inline int get_sha1_with_context(const char *str, unsigned char *sha1, struct object_context *orc)
  {
 -      return get_sha1_with_context_1(str, sha1, orc, 1, NULL);
 +      return get_sha1_with_context_1(str, sha1, orc, 0, NULL);
  }
  extern int get_sha1_hex(const char *hex, unsigned char *sha1);
  extern char *sha1_to_hex(const unsigned char *sha1);  /* static buffer result! */
@@@ -985,7 -968,6 +988,7 @@@ extern struct ref *find_ref_by_name(con
  extern char *git_getpass(const char *prompt);
  extern struct child_process *git_connect(int fd[2], const char *url, const char *prog, int flags);
  extern int finish_connect(struct child_process *conn);
 +extern int git_connection_is_socket(struct child_process *conn);
  extern int path_match(const char *path, int nr, char **match);
  struct extra_have_objects {
        int nr, alloc;
@@@ -1023,16 -1005,6 +1026,16 @@@ extern const char *packed_object_info_d
  /* Dumb servers support */
  extern int update_server_info(int);
  
 +/* git_config_parse_key() returns these negated: */
 +#define CONFIG_INVALID_KEY 1
 +#define CONFIG_NO_SECTION_OR_NAME 2
 +/* git_config_set(), git_config_set_multivar() return the above or these: */
 +#define CONFIG_NO_LOCK -1
 +#define CONFIG_INVALID_FILE 3
 +#define CONFIG_NO_WRITE 4
 +#define CONFIG_NOTHING_SET 5
 +#define CONFIG_INVALID_PATTERN 6
 +
  typedef int (*config_fn_t)(const char *, const char *, void *);
  extern int git_default_config(const char *, const char *, void *);
  extern int git_config_from_file(config_fn_t fn, const char *, void *);

diff --combined sha1_file.c
index 357f9ab7ff6aad644857986f36fd55da8348744f,f0ca6a1749e2517bf3640f9d9f17e866c4eed47f..064a33040812ba8782bf602c693abf08613d6ec7
--- 1/sha1_file.c
--- 2/sha1_file.c
+++ b/sha1_file.c
@@@ -11,6 -11,7 +11,7 @@@
  #include "pack.h"
  #include "blob.h"
  #include "commit.h"
+ #include "run-command.h"
  #include "tag.h"
  #include "tree.h"
  #include "tree-walk.h"
@@@ -31,6 -32,8 +32,6 @@@ static inline uintmax_t sz_fmt(size_t s
  
  const unsigned char null_sha1[20];
  
 -static int git_open_noatime(const char *name, struct packed_git *p);
 -
  /*
   * This is meant to hold a *small* number of objects that you would
   * want read_sha1_file() to be able to return, but yet you do not want
@@@ -225,7 -228,6 +226,7 @@@ struct alternate_object_database *alt_o
  static struct alternate_object_database **alt_odb_tail;
  
  static void read_info_alternates(const char * alternates, int depth);
 +static int git_open_noatime(const char *name);
  
  /*
   * Prepare alternate object database registry.
@@@ -359,7 -361,7 +360,7 @@@ static void read_info_alternates(const 
        int fd;
  
        sprintf(path, "%s/%s", relative_base, alt_file_name);
 -      fd = git_open_noatime(path, NULL);
 +      fd = git_open_noatime(path);
        if (fd < 0)
                return;
        if (fstat(fd, &st) || (st.st_size == 0)) {
@@@ -474,7 -476,7 +475,7 @@@ static int check_packed_git_idx(const c
        struct pack_idx_header *hdr;
        size_t idx_size;
        uint32_t version, nr, i, *index;
 -      int fd = git_open_noatime(path, p);
 +      int fd = git_open_noatime(path);
        struct stat st;
  
        if (fd < 0)
@@@ -756,7 -758,7 +757,7 @@@ static int open_packed_git_1(struct pac
        while (pack_max_fds <= pack_open_fds && unuse_one_window(NULL, -1))
                ; /* nothing */
  
 -      p->pack_fd = git_open_noatime(p->pack_name, p);
 +      p->pack_fd = git_open_noatime(p->pack_name);
        if (p->pack_fd < 0 || fstat(p->pack_fd, &st))
                return -1;
        pack_open_fds++;
@@@ -1144,7 -1146,7 +1145,7 @@@ int check_sha1_signature(const unsigne
        return hashcmp(sha1, real_sha1) ? -1 : 0;
  }
  
 -static int git_open_noatime(const char *name, struct packed_git *p)
 +static int git_open_noatime(const char *name)
  {
        static int sha1_file_open_flag = O_NOATIME;
  
@@@ -1169,7 -1171,7 +1170,7 @@@ static int open_sha1_file(const unsigne
        char *name = sha1_file_name(sha1);
        struct alternate_object_database *alt;
  
 -      fd = git_open_noatime(name, NULL);
 +      fd = git_open_noatime(name);
        if (fd >= 0)
                return fd;
  
@@@ -1178,7 -1180,7 +1179,7 @@@
        for (alt = alt_odb_list; alt; alt = alt->next) {
                name = alt->name;
                fill_sha1_path(name, sha1);
 -              fd = git_open_noatime(alt->base, NULL);
 +              fd = git_open_noatime(alt->base);
                if (fd >= 0)
                        return fd;
        }
@@@ -1307,7 -1309,7 +1308,7 @@@ static void *unpack_sha1_rest(z_stream 
                /*
                 * The above condition must be (bytes <= size), not
                 * (bytes < size).  In other words, even though we
 -               * expect no more output and set avail_out to zer0,
 +               * expect no more output and set avail_out to zero,
                 * the input zlib stream may have bytes that express
                 * "this concludes the stream", and we *do* want to
                 * eat that input.
@@@ -2205,21 -2207,23 +2206,21 @@@ static void *read_object(const unsigne
   * deal with them should arrange to call read_object() and give error
   * messages themselves.
   */
 -void *read_sha1_file_repl(const unsigned char *sha1,
 -                        enum object_type *type,
 -                        unsigned long *size,
 -                        const unsigned char **replacement)
 +void *read_sha1_file_extended(const unsigned char *sha1,
 +                            enum object_type *type,
 +                            unsigned long *size,
 +                            unsigned flag)
  {
 -      const unsigned char *repl = lookup_replace_object(sha1);
        void *data;
        char *path;
        const struct packed_git *p;
 +      const unsigned char *repl = (flag & READ_SHA1_FILE_REPLACE)
 +              ? lookup_replace_object(sha1) : sha1;
  
        errno = 0;
        data = read_object(repl, type, size);
 -      if (data) {
 -              if (replacement)
 -                      *replacement = repl;
 +      if (data)
                return data;
 -      }
  
        if (errno && errno != ENOENT)
                die_errno("failed to read object %s", sha1_to_hex(sha1));
@@@ -2578,10 -2582,11 +2579,11 @@@ static void check_tag(const void *buf, 
  }
  
  static int index_mem(unsigned char *sha1, void *buf, size_t size,
-                    int write_object, enum object_type type,
-                    const char *path, int format_check)
+                    enum object_type type,
+                    const char *path, unsigned flags)
  {
        int ret, re_allocated = 0;
+       int write_object = flags & HASH_WRITE_OBJECT;
  
        if (!type)
                type = OBJ_BLOB;
@@@ -2597,7 -2602,7 +2599,7 @@@
                        re_allocated = 1;
                }
        }
-       if (format_check) {
+       if (flags & HASH_FORMAT_CHECK) {
                if (type == OBJ_TREE)
                        check_tree(buf, size);
                if (type == OBJ_COMMIT)
@@@ -2615,44 -2620,141 +2617,141 @@@
        return ret;
  }
  
+ static int index_pipe(unsigned char *sha1, int fd, enum object_type type,
+                     const char *path, unsigned flags)
+ {
+       struct strbuf sbuf = STRBUF_INIT;
+       int ret;
+ 
+       if (strbuf_read(&sbuf, fd, 4096) >= 0)
+               ret = index_mem(sha1, sbuf.buf, sbuf.len, type, path, flags);
+       else
+               ret = -1;
+       strbuf_release(&sbuf);
+       return ret;
+ }
+ 
  #define SMALL_FILE_SIZE (32*1024)
  
- int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object,
-            enum object_type type, const char *path, int format_check)
+ static int index_core(unsigned char *sha1, int fd, size_t size,
+                     enum object_type type, const char *path,
+                     unsigned flags)
  {
        int ret;
-       size_t size = xsize_t(st->st_size);
  
-       if (!S_ISREG(st->st_mode)) {
-               struct strbuf sbuf = STRBUF_INIT;
-               if (strbuf_read(&sbuf, fd, 4096) >= 0)
-                       ret = index_mem(sha1, sbuf.buf, sbuf.len, write_object,
-                                       type, path, format_check);
-               else
-                       ret = -1;
-               strbuf_release(&sbuf);
-       } else if (!size) {
-               ret = index_mem(sha1, NULL, size, write_object, type, path,
-                               format_check);
+       if (!size) {
+               ret = index_mem(sha1, NULL, size, type, path, flags);
        } else if (size <= SMALL_FILE_SIZE) {
                char *buf = xmalloc(size);
                if (size == read_in_full(fd, buf, size))
-                       ret = index_mem(sha1, buf, size, write_object, type,
-                                       path, format_check);
+                       ret = index_mem(sha1, buf, size, type, path, flags);
                else
                        ret = error("short read %s", strerror(errno));
                free(buf);
        } else {
                void *buf = xmmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
-               ret = index_mem(sha1, buf, size, write_object, type, path,
-                               format_check);
+               ret = index_mem(sha1, buf, size, type, path, flags);
                munmap(buf, size);
        }
+       return ret;
+ }
+ 
+ /*
+  * This creates one packfile per large blob, because the caller
+  * immediately wants the result sha1, and fast-import can report the
+  * object name via marks mechanism only by closing the created
+  * packfile.
+  *
+  * This also bypasses the usual "convert-to-git" dance, and that is on
+  * purpose. We could write a streaming version of the converting
+  * functions and insert that before feeding the data to fast-import
+  * (or equivalent in-core API described above), but the primary
+  * motivation for trying to stream from the working tree file and to
+  * avoid mmaping it in core is to deal with large binary blobs, and
+  * by definition they do _not_ want to get any conversion.
+  */
+ static int index_stream(unsigned char *sha1, int fd, size_t size,
+                       enum object_type type, const char *path,
+                       unsigned flags)
+ {
+       struct child_process fast_import;
+       char export_marks[512];
+       const char *argv[] = { "fast-import", "--quiet", export_marks, NULL };
+       char tmpfile[512];
+       char fast_import_cmd[512];
+       char buf[512];
+       int len, tmpfd;
+ 
+       strcpy(tmpfile, git_path("hashstream_XXXXXX"));
+       tmpfd = git_mkstemp_mode(tmpfile, 0600);
+       if (tmpfd < 0)
+               die_errno("cannot create tempfile: %s", tmpfile);
+       if (close(tmpfd))
+               die_errno("cannot close tempfile: %s", tmpfile);
+       sprintf(export_marks, "--export-marks=%s", tmpfile);
+ 
+       memset(&fast_import, 0, sizeof(fast_import));
+       fast_import.in = -1;
+       fast_import.argv = argv;
+       fast_import.git_cmd = 1;
+       if (start_command(&fast_import))
+               die_errno("index-stream: git fast-import failed");
+ 
+       len = sprintf(fast_import_cmd, "blob\nmark :1\ndata %lu\n",
+                     (unsigned long) size);
+       write_or_whine(fast_import.in, fast_import_cmd, len,
+                      "index-stream: feeding fast-import");
+       while (size) {
+               char buf[10240];
+               size_t sz = size < sizeof(buf) ? size : sizeof(buf);
+               size_t actual;
+ 
+               actual = read_in_full(fd, buf, sz);
+               if (actual < 0)
+                       die_errno("index-stream: reading input");
+               if (write_in_full(fast_import.in, buf, actual) != actual)
+                       die_errno("index-stream: feeding fast-import");
+               size -= actual;
+       }
+       if (close(fast_import.in))
+               die_errno("index-stream: closing fast-import");
+       if (finish_command(&fast_import))
+               die_errno("index-stream: finishing fast-import");
+ 
+       tmpfd = open(tmpfile, O_RDONLY);
+       if (tmpfd < 0)
+               die_errno("index-stream: cannot open fast-import mark");
+       len = read(tmpfd, buf, sizeof(buf));
+       if (len < 0)
+               die_errno("index-stream: reading fast-import mark");
+       if (close(tmpfd) < 0)
+               die_errno("index-stream: closing fast-import mark");
+       if (unlink(tmpfile))
+               die_errno("index-stream: unlinking fast-import mark");
+       if (len != 44 ||
+           memcmp(":1 ", buf, 3) ||
+           get_sha1_hex(buf + 3, sha1))
+               die_errno("index-stream: unexpected fast-import mark: <%s>", buf);
+       return 0;
+ }
+ 
+ int index_fd(unsigned char *sha1, int fd, struct stat *st,
+            enum object_type type, const char *path, unsigned flags)
+ {
+       int ret;
+       size_t size = xsize_t(st->st_size);
+ 
+       if (!S_ISREG(st->st_mode))
+               ret = index_pipe(sha1, fd, type, path, flags);
+       else if (size <= big_file_threshold || type != OBJ_BLOB)
+               ret = index_core(sha1, fd, size, type, path, flags);
+       else
+               ret = index_stream(sha1, fd, size, type, path, flags);
        close(fd);
        return ret;
  }
  
- int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object)
+ int index_path(unsigned char *sha1, const char *path, struct stat *st, unsigned flags)
  {
        int fd;
        struct strbuf sb = STRBUF_INIT;
@@@ -2663,7 -2765,7 +2762,7 @@@
                if (fd < 0)
                        return error("open(\"%s\"): %s", path,
                                     strerror(errno));
-               if (index_fd(sha1, fd, st, write_object, OBJ_BLOB, path, 0) < 0)
+               if (index_fd(sha1, fd, st, OBJ_BLOB, path, flags) < 0)
                        return error("%s: failed to insert into database",
                                     path);
                break;
@@@ -2673,7 -2775,7 +2772,7 @@@
                        return error("readlink(\"%s\"): %s", path,
                                     errstr);
                }
-               if (!write_object)
+               if (!(flags & HASH_WRITE_OBJECT))
                        hash_sha1_file(sb.buf, sb.len, blob_type, sha1);
                else if (write_sha1_file(sb.buf, sb.len, blob_type, sha1))
                        return error("%s: failed to insert into database",