Code

reduce delta head inflated size
[git.git] / sha1_file.c
index 769a80984d25ca362acb8d7c4f97ac3526c2b753..47e2a29abd6d2ae067d72c0f73bcf4e6afe0a938 100644 (file)
@@ -26,16 +26,40 @@ const unsigned char null_sha1[20];
 
 static unsigned int sha1_file_open_flag = O_NOATIME;
 
-static unsigned hexval(char c)
-{
-       if (c >= '0' && c <= '9')
-               return c - '0';
-       if (c >= 'a' && c <= 'f')
-               return c - 'a' + 10;
-       if (c >= 'A' && c <= 'F')
-               return c - 'A' + 10;
-       return ~0;
-}
+signed char hexval_table[256] = {
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 00-07 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 08-0f */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 10-17 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 18-1f */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 20-27 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 28-2f */
+         0,  1,  2,  3,  4,  5,  6,  7,                /* 30-37 */
+         8,  9, -1, -1, -1, -1, -1, -1,                /* 38-3f */
+        -1, 10, 11, 12, 13, 14, 15, -1,                /* 40-47 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 48-4f */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 50-57 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 58-5f */
+        -1, 10, 11, 12, 13, 14, 15, -1,                /* 60-67 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 68-67 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 70-77 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 78-7f */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 80-87 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 88-8f */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 90-97 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* 98-9f */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* a0-a7 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* a8-af */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* b0-b7 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* b8-bf */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* c0-c7 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* c8-cf */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* d0-d7 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* d8-df */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* e0-e7 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* e8-ef */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* f0-f7 */
+        -1, -1, -1, -1, -1, -1, -1, -1,                /* f8-ff */
+};
 
 int get_sha1_hex(const char *hex, unsigned char *sha1)
 {
@@ -115,7 +139,7 @@ static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
 
 /*
  * NOTE! This returns a statically allocated buffer, so you have to be
- * careful about using it. Do a "strdup()" if you need to save the
+ * careful about using it. Do a "xstrdup()" if you need to save the
  * filename.
  *
  * Also note that this returns the location for creating.  Reading
@@ -647,14 +671,8 @@ static void reprepare_packed_git(void)
 
 int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long size, const char *type)
 {
-       char header[100];
        unsigned char real_sha1[20];
-       SHA_CTX c;
-
-       SHA1_Init(&c);
-       SHA1_Update(&c, header, 1+sprintf(header, "%s %lu", type, size));
-       SHA1_Update(&c, map, size);
-       SHA1_Final(real_sha1, &c);
+       hash_sha1_file(map, size, type, real_sha1);
        return hashcmp(sha1, real_sha1) ? -1 : 0;
 }
 
@@ -711,17 +729,39 @@ int legacy_loose_object(unsigned char *map)
                return 0;
 }
 
-static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
+unsigned long unpack_object_header_gently(const unsigned char *buf, unsigned long len, enum object_type *type, unsigned long *sizep)
 {
+       unsigned shift;
        unsigned char c;
-       unsigned int bits;
        unsigned long size;
-       static const char *typename[8] = {
-               NULL,   /* OBJ_EXT */
-               "commit", "tree", "blob", "tag",
-               NULL, NULL, NULL
+       unsigned long used = 0;
+
+       c = buf[used++];
+       *type = (c >> 4) & 7;
+       size = c & 15;
+       shift = 4;
+       while (c & 0x80) {
+               if (len <= used)
+                       return 0;
+               if (sizeof(long) * 8 <= shift)
+                       return 0;
+               c = buf[used++];
+               size += (c & 0x7f) << shift;
+               shift += 7;
+       }
+       *sizep = size;
+       return used;
+}
+
+static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
+{
+       unsigned long size, used;
+       static const char valid_loose_object_type[8] = {
+               0, /* OBJ_EXT */
+               1, 1, 1, 1, /* "commit", "tree", "blob", "tag" */
+               0, /* "delta" and others are invalid in a loose object */
        };
-       const char *type;
+       enum object_type type;
 
        /* Get the data stream */
        memset(stream, 0, sizeof(*stream));
@@ -735,22 +775,11 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon
                return inflate(stream, 0);
        }
 
-       c = *map++;
-       mapsize--;
-       type = typename[(c >> 4) & 7];
-       if (!type)
+       used = unpack_object_header_gently(map, mapsize, &type, &size);
+       if (!used || !valid_loose_object_type[type])
                return -1;
-
-       bits = 4;
-       size = c & 0xf;
-       while ((c & 0x80)) {
-               if (bits >= 8*sizeof(long))
-                       return -1;
-               c = *map++;
-               size += (c & 0x7f) << bits;
-               bits += 7;
-               mapsize--;
-       }
+       map += used;
+       mapsize -= used;
 
        /* Set up the stream for the rest.. */
        stream->next_in = map;
@@ -758,7 +787,8 @@ static int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned lon
        inflateInit(stream);
 
        /* And generate the fake traditional header */
-       stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu", type, size);
+       stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
+                                        type_names[type], size);
        return 0;
 }
 
@@ -848,46 +878,45 @@ void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned l
 }
 
 /* forward declaration for a mutually recursive function */
-static int packed_object_info(struct pack_entry *entry,
+static int packed_object_info(struct packed_git *p, unsigned long offset,
                              char *type, unsigned long *sizep);
 
-static int packed_delta_info(unsigned char *base_sha1,
-                            unsigned long delta_size,
-                            unsigned long left,
+static int packed_delta_info(struct packed_git *p,
+                            unsigned long offset,
                             char *type,
-                            unsigned long *sizep,
-                            struct packed_git *p)
+                            unsigned long *sizep)
 {
-       struct pack_entry base_ent;
+       unsigned long base_offset;
+       unsigned char *base_sha1 = (unsigned char *) p->pack_base + offset;
 
-       if (left < 20)
+       if (p->pack_size < offset + 20)
                die("truncated pack file");
-
        /* The base entry _must_ be in the same pack */
-       if (!find_pack_entry_one(base_sha1, &base_ent, p))
+       base_offset = find_pack_entry_one(base_sha1, p);
+       if (!base_offset)
                die("failed to find delta-pack base object %s",
                    sha1_to_hex(base_sha1));
+       offset += 20;
 
        /* We choose to only get the type of the base object and
         * ignore potentially corrupt pack file that expects the delta
         * based on a base with a wrong size.  This saves tons of
         * inflate() calls.
         */
-
-       if (packed_object_info(&base_ent, type, NULL))
+       if (packed_object_info(p, base_offset, type, NULL))
                die("cannot get info for delta-pack base");
 
        if (sizep) {
                const unsigned char *data;
-               unsigned char delta_head[64];
+               unsigned char delta_head[20];
                unsigned long result_size;
                z_stream stream;
                int st;
 
                memset(&stream, 0, sizeof(stream));
 
-               data = stream.next_in = base_sha1 + 20;
-               stream.avail_in = left - 20;
+               stream.next_in = (unsigned char *) p->pack_base + offset;
+               stream.avail_in = p->pack_size - offset;
                stream.next_out = delta_head;
                stream.avail_out = sizeof(delta_head);
 
@@ -916,29 +945,18 @@ static int packed_delta_info(unsigned char *base_sha1,
 static unsigned long unpack_object_header(struct packed_git *p, unsigned long offset,
        enum object_type *type, unsigned long *sizep)
 {
-       unsigned shift;
-       unsigned char *pack, c;
-       unsigned long size;
+       unsigned long used;
 
-       if (offset >= p->pack_size)
+       if (p->pack_size <= offset)
                die("object offset outside of pack file");
 
-       pack =  (unsigned char *) p->pack_base + offset;
-       c = *pack++;
-       offset++;
-       *type = (c >> 4) & 7;
-       size = c & 15;
-       shift = 4;
-       while (c & 0x80) {
-               if (offset >= p->pack_size)
-                       die("object offset outside of pack file");
-               c = *pack++;
-               offset++;
-               size += (c & 0x7f) << shift;
-               shift += 7;
-       }
-       *sizep = size;
-       return offset;
+       used = unpack_object_header_gently((unsigned char *)p->pack_base +
+                                          offset,
+                                          p->pack_size - offset, type, sizep);
+       if (!used)
+               die("object offset outside of pack file");
+
+       return offset + used;
 }
 
 int check_reuse_pack_delta(struct packed_git *p, unsigned long offset,
@@ -960,92 +978,65 @@ int check_reuse_pack_delta(struct packed_git *p, unsigned long offset,
        return status;
 }
 
-void packed_object_info_detail(struct pack_entry *e,
+void packed_object_info_detail(struct packed_git *p,
+                              unsigned long offset,
                               char *type,
                               unsigned long *size,
                               unsigned long *store_size,
                               unsigned int *delta_chain_length,
                               unsigned char *base_sha1)
 {
-       struct packed_git *p = e->p;
-       unsigned long offset;
-       unsigned char *pack;
+       unsigned long val;
+       unsigned char *next_sha1;
        enum object_type kind;
 
-       offset = unpack_object_header(p, e->offset, &kind, size);
-       pack = (unsigned char *) p->pack_base + offset;
-       if (kind != OBJ_DELTA)
-               *delta_chain_length = 0;
-       else {
-               unsigned int chain_length = 0;
-               if (p->pack_size <= offset + 20)
-                       die("pack file %s records an incomplete delta base",
-                           p->pack_name);
-               hashcpy(base_sha1, pack);
-               do {
-                       struct pack_entry base_ent;
-                       unsigned long junk;
-
-                       find_pack_entry_one(pack, &base_ent, p);
-                       offset = unpack_object_header(p, base_ent.offset,
-                                                     &kind, &junk);
-                       pack = (unsigned char *) p->pack_base + offset;
-                       chain_length++;
-               } while (kind == OBJ_DELTA);
-               *delta_chain_length = chain_length;
-       }
-       switch (kind) {
-       case OBJ_COMMIT:
-               strcpy(type, commit_type);
-               break;
-       case OBJ_TREE:
-               strcpy(type, tree_type);
-               break;
-       case OBJ_BLOB:
-               strcpy(type, blob_type);
-               break;
-       case OBJ_TAG:
-               strcpy(type, tag_type);
-               break;
-       default:
-               die("corrupted pack file %s containing object of kind %d",
-                   p->pack_name, kind);
+       *delta_chain_length = 0;
+       offset = unpack_object_header(p, offset, &kind, size);
+
+       for (;;) {
+               switch (kind) {
+               default:
+                       die("corrupted pack file %s containing object of kind %d",
+                           p->pack_name, kind);
+               case OBJ_COMMIT:
+               case OBJ_TREE:
+               case OBJ_BLOB:
+               case OBJ_TAG:
+                       strcpy(type, type_names[kind]);
+                       *store_size = 0; /* notyet */
+                       return;
+               case OBJ_DELTA:
+                       if (p->pack_size <= offset + 20)
+                               die("pack file %s records an incomplete delta base",
+                                   p->pack_name);
+                       next_sha1 = (unsigned char *) p->pack_base + offset;
+                       if (*delta_chain_length == 0)
+                               hashcpy(base_sha1, next_sha1);
+                       offset = find_pack_entry_one(next_sha1, p);
+                       break;
+               }
+               offset = unpack_object_header(p, offset, &kind, &val);
+               (*delta_chain_length)++;
        }
-       *store_size = 0; /* notyet */
 }
 
-static int packed_object_info(struct pack_entry *entry,
+static int packed_object_info(struct packed_git *p, unsigned long offset,
                              char *type, unsigned long *sizep)
 {
-       struct packed_git *p = entry->p;
-       unsigned long offset, size, left;
-       unsigned char *pack;
+       unsigned long size;
        enum object_type kind;
-       int retval;
 
-       if (use_packed_git(p))
-               die("cannot map packed file");
+       offset = unpack_object_header(p, offset, &kind, &size);
 
-       offset = unpack_object_header(p, entry->offset, &kind, &size);
-       pack = (unsigned char *) p->pack_base + offset;
-       left = p->pack_size - offset;
+       if (kind == OBJ_DELTA)
+               return packed_delta_info(p, offset, type, sizep);
 
        switch (kind) {
-       case OBJ_DELTA:
-               retval = packed_delta_info(pack, size, left, type, sizep, p);
-               unuse_packed_git(p);
-               return retval;
        case OBJ_COMMIT:
-               strcpy(type, commit_type);
-               break;
        case OBJ_TREE:
-               strcpy(type, tree_type);
-               break;
        case OBJ_BLOB:
-               strcpy(type, blob_type);
-               break;
        case OBJ_TAG:
-               strcpy(type, tag_type);
+               strcpy(type, type_names[kind]);
                break;
        default:
                die("corrupted pack file %s containing object of kind %d",
@@ -1053,66 +1044,12 @@ static int packed_object_info(struct pack_entry *entry,
        }
        if (sizep)
                *sizep = size;
-       unuse_packed_git(p);
        return 0;
 }
 
-static void *unpack_delta_entry(unsigned char *base_sha1,
-                               unsigned long delta_size,
-                               unsigned long left,
-                               char *type,
-                               unsigned long *sizep,
-                               struct packed_git *p)
-{
-       struct pack_entry base_ent;
-       void *data, *delta_data, *result, *base;
-       unsigned long data_size, result_size, base_size;
-       z_stream stream;
-       int st;
-
-       if (left < 20)
-               die("truncated pack file");
-
-       /* The base entry _must_ be in the same pack */
-       if (!find_pack_entry_one(base_sha1, &base_ent, p))
-               die("failed to find delta-pack base object %s",
-                   sha1_to_hex(base_sha1));
-       base = unpack_entry_gently(&base_ent, type, &base_size);
-       if (!base)
-               die("failed to read delta-pack base object %s",
-                   sha1_to_hex(base_sha1));
-
-       data = base_sha1 + 20;
-       data_size = left - 20;
-       delta_data = xmalloc(delta_size);
-
-       memset(&stream, 0, sizeof(stream));
-
-       stream.next_in = data;
-       stream.avail_in = data_size;
-       stream.next_out = delta_data;
-       stream.avail_out = delta_size;
-
-       inflateInit(&stream);
-       st = inflate(&stream, Z_FINISH);
-       inflateEnd(&stream);
-       if ((st != Z_STREAM_END) || stream.total_out != delta_size)
-               die("delta data unpack failed");
-
-       result = patch_delta(base, base_size,
-                            delta_data, delta_size,
-                            &result_size);
-       if (!result)
-               die("failed to apply delta");
-       free(delta_data);
-       free(base);
-       *sizep = result_size;
-       return result;
-}
-
-static void *unpack_non_delta_entry(unsigned char *data,
-                                   unsigned long size,
-                                   unsigned long left)
+static void *unpack_compressed_entry(struct packed_git *p,
+                                   unsigned long offset,
+                                   unsigned long size)
 {
        int st;
        z_stream stream;
@@ -1121,8 +1058,8 @@ static void *unpack_non_delta_entry(unsigned char *data,
        buffer = xmalloc(size + 1);
        buffer[size] = 0;
        memset(&stream, 0, sizeof(stream));
-       stream.next_in = data;
-       stream.avail_in = left;
+       stream.next_in = (unsigned char*)p->pack_base + offset;
+       stream.avail_in = p->pack_size - offset;
        stream.next_out = buffer;
        stream.avail_out = size;
 
@@ -1137,6 +1074,43 @@ static void *unpack_non_delta_entry(unsigned char *data,
        return buffer;
 }
 
+static void *unpack_delta_entry(struct packed_git *p,
+                               unsigned long offset,
+                               unsigned long delta_size,
+                               char *type,
+                               unsigned long *sizep)
+{
+       void *delta_data, *result, *base;
+       unsigned long result_size, base_size, base_offset;
+       unsigned char *base_sha1;
+
+       if (p->pack_size < offset + 20)
+               die("truncated pack file");
+       /* The base entry _must_ be in the same pack */
+       base_sha1 = (unsigned char*)p->pack_base + offset;
+       base_offset = find_pack_entry_one(base_sha1, p);
+       if (!base_offset)
+               die("failed to find delta-pack base object %s",
+                   sha1_to_hex(base_sha1));
+       offset += 20;
+
+       base = unpack_entry_gently(p, base_offset, type, &base_size);
+       if (!base)
+               die("failed to read delta base object at %lu from %s",
+                   base_offset, p->pack_name);
+
+       delta_data = unpack_compressed_entry(p, offset, delta_size);
+       result = patch_delta(base, base_size,
+                            delta_data, delta_size,
+                            &result_size);
+       if (!result)
+               die("failed to apply delta");
+       free(delta_data);
+       free(base);
+       *sizep = result_size;
+       return result;
+}
+
 static void *unpack_entry(struct pack_entry *entry,
                          char *type, unsigned long *sizep)
 {
@@ -1145,7 +1119,7 @@ static void *unpack_entry(struct pack_entry *entry,
 
        if (use_packed_git(p))
                die("cannot map packed file");
-       retval = unpack_entry_gently(entry, type, sizep);
+       retval = unpack_entry_gently(p, entry->offset, type, sizep);
        unuse_packed_git(p);
        if (!retval)
                die("corrupted pack file %s", p->pack_name);
@@ -1153,40 +1127,26 @@ static void *unpack_entry(struct pack_entry *entry,
 }
 
 /* The caller is responsible for use_packed_git()/unuse_packed_git() pair */
-void *unpack_entry_gently(struct pack_entry *entry,
+void *unpack_entry_gently(struct packed_git *p, unsigned long offset,
                          char *type, unsigned long *sizep)
 {
-       struct packed_git *p = entry->p;
-       unsigned long offset, size, left;
-       unsigned char *pack;
+       unsigned long size;
        enum object_type kind;
-       void *retval;
 
-       offset = unpack_object_header(p, entry->offset, &kind, &size);
-       pack = (unsigned char *) p->pack_base + offset;
-       left = p->pack_size - offset;
+       offset = unpack_object_header(p, offset, &kind, &size);
        switch (kind) {
        case OBJ_DELTA:
-               retval = unpack_delta_entry(pack, size, left, type, sizep, p);
-               return retval;
+               return unpack_delta_entry(p, offset, size, type, sizep);
        case OBJ_COMMIT:
-               strcpy(type, commit_type);
-               break;
        case OBJ_TREE:
-               strcpy(type, tree_type);
-               break;
        case OBJ_BLOB:
-               strcpy(type, blob_type);
-               break;
        case OBJ_TAG:
-               strcpy(type, tag_type);
-               break;
+               strcpy(type, type_names[kind]);
+               *sizep = size;
+               return unpack_compressed_entry(p, offset, size);
        default:
                return NULL;
        }
-       *sizep = size;
-       retval = unpack_non_delta_entry(pack, size, left);
-       return retval;
 }
 
 int num_packed_objects(const struct packed_git *p)
@@ -1205,8 +1165,8 @@ int nth_packed_object_sha1(const struct packed_git *p, int n,
        return 0;
 }
 
-int find_pack_entry_one(const unsigned char *sha1,
-                       struct pack_entry *e, struct packed_git *p)
+unsigned long find_pack_entry_one(const unsigned char *sha1,
+                                 struct packed_git *p)
 {
        unsigned int *level1_ofs = p->index_base;
        int hi = ntohl(level1_ofs[*sha1]);
@@ -1216,12 +1176,8 @@ int find_pack_entry_one(const unsigned char *sha1,
        do {
                int mi = (lo + hi) / 2;
                int cmp = hashcmp((unsigned char *)index + (24 * mi) + 4, sha1);
-               if (!cmp) {
-                       e->offset = ntohl(*((unsigned int *) ((char *) index + (24 * mi))));
-                       hashcpy(e->sha1, sha1);
-                       e->p = p;
-                       return 1;
-               }
+               if (!cmp)
+                       return ntohl(*((unsigned int *) ((char *) index + (24 * mi))));
                if (cmp > 0)
                        hi = mi;
                else
@@ -1230,14 +1186,29 @@ int find_pack_entry_one(const unsigned char *sha1,
        return 0;
 }
 
-static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
+static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e, const char **ignore_packed)
 {
        struct packed_git *p;
+       unsigned long offset;
+
        prepare_packed_git();
 
        for (p = packed_git; p; p = p->next) {
-               if (find_pack_entry_one(sha1, e, p))
+               if (ignore_packed) {
+                       const char **ig;
+                       for (ig = ignore_packed; *ig; ig++)
+                               if (!strcmp(p->pack_name, *ig))
+                                       break;
+                       if (*ig)
+                               continue;
+               }
+               offset = find_pack_entry_one(sha1, p);
+               if (offset) {
+                       e->offset = offset;
+                       e->p = p;
+                       hashcpy(e->sha1, sha1);
                        return 1;
+               }
        }
        return 0;
 }
@@ -1246,10 +1217,9 @@ struct packed_git *find_sha1_pack(const unsigned char *sha1,
                                  struct packed_git *packs)
 {
        struct packed_git *p;
-       struct pack_entry e;
 
        for (p = packs; p; p = p->next) {
-               if (find_pack_entry_one(sha1, &e, p))
+               if (find_pack_entry_one(sha1, p))
                        return p;
        }
        return NULL;
@@ -1268,12 +1238,16 @@ int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep
        if (!map) {
                struct pack_entry e;
 
-               if (find_pack_entry(sha1, &e))
-                       return packed_object_info(&e, type, sizep);
-               reprepare_packed_git();
-               if (find_pack_entry(sha1, &e))
-                       return packed_object_info(&e, type, sizep);
-               return error("unable to find %s", sha1_to_hex(sha1));
+               if (!find_pack_entry(sha1, &e, NULL)) {
+                       reprepare_packed_git();
+                       if (!find_pack_entry(sha1, &e, NULL))
+                               return error("unable to find %s", sha1_to_hex(sha1));
+               }
+               if (use_packed_git(e.p))
+                       die("cannot map packed file");
+               status = packed_object_info(e.p, e.offset, type, sizep);
+               unuse_packed_git(e.p);
+               return status;
        }
        if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
                status = error("unable to unpack %s header",
@@ -1294,7 +1268,7 @@ static void *read_packed_sha1(const unsigned char *sha1, char *type, unsigned lo
 {
        struct pack_entry e;
 
-       if (!find_pack_entry(sha1, &e)) {
+       if (!find_pack_entry(sha1, &e, NULL)) {
                error("cannot read sha1_file for %s", sha1_to_hex(sha1));
                return NULL;
        }
@@ -1307,7 +1281,7 @@ void * read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size
        void *map, *buf;
        struct pack_entry e;
 
-       if (find_pack_entry(sha1, &e))
+       if (find_pack_entry(sha1, &e, NULL))
                return read_packed_sha1(sha1, type, size);
        map = map_sha1_file(sha1, &mapsize);
        if (map) {
@@ -1316,7 +1290,7 @@ void * read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size
                return buf;
        }
        reprepare_packed_git();
-       if (find_pack_entry(sha1, &e))
+       if (find_pack_entry(sha1, &e, NULL))
                return read_packed_sha1(sha1, type, size);
        return NULL;
 }
@@ -1367,12 +1341,9 @@ void *read_object_with_reference(const unsigned char *sha1,
        }
 }
 
-char *write_sha1_file_prepare(void *buf,
-                             unsigned long len,
-                             const char *type,
-                             unsigned char *sha1,
-                             unsigned char *hdr,
-                             int *hdrlen)
+static void write_sha1_file_prepare(void *buf, unsigned long len,
+                                    const char *type, unsigned char *sha1,
+                                    unsigned char *hdr, int *hdrlen)
 {
        SHA_CTX c;
 
@@ -1384,8 +1355,6 @@ char *write_sha1_file_prepare(void *buf,
        SHA1_Update(&c, hdr, *hdrlen);
        SHA1_Update(&c, buf, len);
        SHA1_Final(sha1, &c);
-
-       return sha1_file_name(sha1);
 }
 
 /*
@@ -1394,7 +1363,7 @@ char *write_sha1_file_prepare(void *buf,
  *
  * Returns the errno on failure, 0 on success.
  */
-static int link_temp_to_file(const char *tmpfile, char *filename)
+static int link_temp_to_file(const char *tmpfile, const char *filename)
 {
        int ret;
        char *dir;
@@ -1427,7 +1396,7 @@ static int link_temp_to_file(const char *tmpfile, char *filename)
 /*
  * Move the just written object into its final resting place
  */
-int move_temp_to_file(const char *tmpfile, char *filename)
+int move_temp_to_file(const char *tmpfile, const char *filename)
 {
        int ret = link_temp_to_file(tmpfile, filename);
 
@@ -1521,6 +1490,15 @@ static void setup_object_header(z_stream *stream, const char *type, unsigned lon
        stream->avail_out -= hdr;
 }
 
+int hash_sha1_file(void *buf, unsigned long len, const char *type,
+                   unsigned char *sha1)
+{
+       unsigned char hdr[50];
+       int hdrlen;
+       write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
+       return 0;
+}
+
 int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
 {
        int size;
@@ -1535,7 +1513,8 @@ int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned cha
        /* Normally if we have it in the pack then we do not bother writing
         * it out into .git/objects/??/?{38} file.
         */
-       filename = write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
+       write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
+       filename = sha1_file_name(sha1);
        if (returnsha1)
                hashcpy(returnsha1, sha1);
        if (has_sha1_file(sha1))
@@ -1748,10 +1727,10 @@ int has_pack_file(const unsigned char *sha1)
        return 1;
 }
 
-int has_sha1_pack(const unsigned char *sha1)
+int has_sha1_pack(const unsigned char *sha1, const char **ignore_packed)
 {
        struct pack_entry e;
-       return find_pack_entry(sha1, &e);
+       return find_pack_entry(sha1, &e, ignore_packed);
 }
 
 int has_sha1_file(const unsigned char *sha1)
@@ -1759,7 +1738,7 @@ int has_sha1_file(const unsigned char *sha1)
        struct stat st;
        struct pack_entry e;
 
-       if (find_pack_entry(sha1, &e))
+       if (find_pack_entry(sha1, &e, NULL))
                return 1;
        return find_sha1_file(sha1, &st) ? 1 : 0;
 }
@@ -1786,7 +1765,7 @@ int read_pipe(int fd, char** return_buf, unsigned long* return_size)
                        off += iret;
                        if (off == size) {
                                size *= 2;
-                               buf = realloc(buf, size);
+                               buf = xrealloc(buf, size);
                        }
                }
        } while (iret > 0);
@@ -1802,10 +1781,8 @@ int read_pipe(int fd, char** return_buf, unsigned long* return_size)
 int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object)
 {
        unsigned long size = 4096;
-       char *buf = malloc(size);
+       char *buf = xmalloc(size);
        int ret;
-       unsigned char hdr[50];
-       int hdrlen;
 
        if (read_pipe(fd, &buf, &size)) {
                free(buf);
@@ -1816,10 +1793,8 @@ int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object)
                type = blob_type;
        if (write_object)
                ret = write_sha1_file(buf, size, type, sha1);
-       else {
-               write_sha1_file_prepare(buf, size, type, sha1, hdr, &hdrlen);
-               ret = 0;
-       }
+       else
+               ret = hash_sha1_file(buf, size, type, sha1);
        free(buf);
        return ret;
 }
@@ -1829,8 +1804,6 @@ int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, con
        unsigned long size = st->st_size;
        void *buf;
        int ret;
-       unsigned char hdr[50];
-       int hdrlen;
 
        buf = "";
        if (size)
@@ -1843,10 +1816,8 @@ int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, con
                type = blob_type;
        if (write_object)
                ret = write_sha1_file(buf, size, type, sha1);
-       else {
-               write_sha1_file_prepare(buf, size, type, sha1, hdr, &hdrlen);
-               ret = 0;
-       }
+       else
+               ret = hash_sha1_file(buf, size, type, sha1);
        if (size)
                munmap(buf, size);
        return ret;
@@ -1875,12 +1846,9 @@ int index_path(unsigned char *sha1, const char *path, struct stat *st, int write
                        return error("readlink(\"%s\"): %s", path,
                                     errstr);
                }
-               if (!write_object) {
-                       unsigned char hdr[50];
-                       int hdrlen;
-                       write_sha1_file_prepare(target, st->st_size, blob_type,
-                                               sha1, hdr, &hdrlen);
-               } else if (write_sha1_file(target, st->st_size, blob_type, sha1))
+               if (!write_object)
+                       hash_sha1_file(target, st->st_size, blob_type, sha1);
+               else if (write_sha1_file(target, st->st_size, blob_type, sha1))
                        return error("%s: failed to insert into database",
                                     path);
                free(target);