![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index 807be8c3f8a051d9171cf2ee3c9f973f77a2ff08..f8ebad0b2f2006d619dc06814acb6024ee674eec 100644 (file)
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
git-pack-objects [{ -q | --progress | --all-progress }] \n\
[--local] [--incremental] [--window=N] [--depth=N] \n\
[--no-reuse-delta] [--delta-base-offset] [--non-empty] \n\
- [--revs [--unpacked | --all]*] [--stdout | base-name] \n\
+ [--revs [--unpacked | --all]*] [--reflog] [--stdout | base-name] \n\
[<ref-list | <object-list]";
struct object_entry {
unsigned char sha1[20];
unsigned long size; /* uncompressed size */
- unsigned long offset; /* offset into the final pack file;
+ off_t offset; /* offset into the final pack file;
* nonzero if already written.
*/
unsigned int depth; /* delta depth */
#define in_pack_header_size delta_size /* only when reusing pack data */
struct object_entry *delta; /* delta base object */
struct packed_git *in_pack; /* already in pack */
- unsigned int in_pack_offset;
+ off_t in_pack_offset;
struct object_entry *delta_child; /* deltified objects who bases me */
struct object_entry *delta_sibling; /* other deltified objects who
* uses the same base as me
static struct object_entry **sorted_by_sha, **sorted_by_type;
static struct object_entry *objects;
-static int nr_objects, nr_alloc, nr_result;
+static uint32_t nr_objects, nr_alloc, nr_result;
static const char *base_name;
static unsigned char pack_file_sha1[20];
static int progress = 1;
* get the object sha1 from the main index.
*/
struct revindex_entry {
- unsigned int offset;
+ off_t offset;
unsigned int nr;
};
struct pack_revindex {
/*
* stats
*/
-static int written;
-static int written_delta;
-static int reused;
-static int reused_delta;
+static uint32_t written, written_delta;
+static uint32_t reused, reused_delta;
static int pack_revindex_ix(struct packed_git *p)
{
}
static struct revindex_entry * find_packed_object(struct packed_git *p,
- unsigned int ofs)
+ off_t ofs)
{
int num;
int lo, hi;
die("internal error: pack revindex corrupt");
}
-static unsigned long find_packed_object_size(struct packed_git *p,
- unsigned long ofs)
+static off_t find_packed_object_size(struct packed_git *p, off_t ofs)
{
struct revindex_entry *entry = find_packed_object(p, ofs);
return entry[1].offset - ofs;
}
static unsigned char *find_packed_object_name(struct packed_git *p,
- unsigned long ofs)
+ off_t ofs)
{
struct revindex_entry *entry = find_packed_object(p, ofs);
return (unsigned char *)(p->index_base + 256) + 24 * entry->nr + 4;
static void *delta_against(void *buf, unsigned long size, struct object_entry *entry)
{
unsigned long othersize, delta_size;
- char type[10];
- void *otherbuf = read_sha1_file(entry->delta->sha1, type, &othersize);
+ enum object_type type;
+ void *otherbuf = read_sha1_file(entry->delta->sha1, &type, &othersize);
void *delta_buf;
if (!otherbuf)
@@ -276,7 +273,52 @@ static int encode_header(enum object_type type, unsigned long size, unsigned cha
* we are going to reuse the existing object data as is. make
* sure it is not corrupt.
*/
-static int check_inflate(unsigned char *data, unsigned long len, unsigned long expect)
+static int check_pack_inflate(struct packed_git *p,
+ struct pack_window **w_curs,
+ off_t offset,
+ off_t len,
+ unsigned long expect)
+{
+ z_stream stream;
+ unsigned char fakebuf[4096], *in;
+ int st;
+
+ memset(&stream, 0, sizeof(stream));
+ inflateInit(&stream);
+ do {
+ in = use_pack(p, w_curs, offset, &stream.avail_in);
+ stream.next_in = in;
+ stream.next_out = fakebuf;
+ stream.avail_out = sizeof(fakebuf);
+ st = inflate(&stream, Z_FINISH);
+ offset += stream.next_in - in;
+ } while (st == Z_OK || st == Z_BUF_ERROR);
+ inflateEnd(&stream);
+ return (st == Z_STREAM_END &&
+ stream.total_out == expect &&
+ stream.total_in == len) ? 0 : -1;
+}
+
+static void copy_pack_data(struct sha1file *f,
+ struct packed_git *p,
+ struct pack_window **w_curs,
+ off_t offset,
+ off_t len)
+{
+ unsigned char *in;
+ unsigned int avail;
+
+ while (len) {
+ in = use_pack(p, w_curs, offset, &avail);
+ if (avail > len)
+ avail = (unsigned int)len;
+ sha1write(f, in, avail);
+ offset += avail;
+ len -= avail;
+ }
+}
+
+static int check_loose_inflate(unsigned char *data, unsigned long len, unsigned long expect)
{
z_stream stream;
unsigned char fakebuf[4096];
return -1;
map += used;
mapsize -= used;
- return check_inflate(map, mapsize, size);
+ return check_loose_inflate(map, mapsize, size);
}
-static unsigned long write_object(struct sha1file *f,
+static off_t write_object(struct sha1file *f,
struct object_entry *entry)
{
unsigned long size;
- char type[10];
+ enum object_type type;
void *buf;
unsigned char header[10];
- unsigned hdrlen, datalen;
+ unsigned hdrlen;
+ off_t datalen;
enum object_type obj_type;
int to_reuse = 0;
}
if (!to_reuse) {
- buf = read_sha1_file(entry->sha1, type, &size);
+ buf = read_sha1_file(entry->sha1, &type, &size);
if (!buf)
die("unable to read %s", sha1_to_hex(entry->sha1));
if (size != entry->size)
* encoding of the relative offset for the delta
* base from this object's position in the pack.
*/
- unsigned long ofs = entry->offset - entry->delta->offset;
+ off_t ofs = entry->offset - entry->delta->offset;
unsigned pos = sizeof(header) - 1;
header[pos] = ofs & 127;
while (ofs >>= 7)
}
else {
struct packed_git *p = entry->in_pack;
+ struct pack_window *w_curs = NULL;
+ off_t offset;
if (entry->delta) {
obj_type = (allow_ofs_delta && entry->delta->offset) ?
hdrlen = encode_header(obj_type, entry->size, header);
sha1write(f, header, hdrlen);
if (obj_type == OBJ_OFS_DELTA) {
- unsigned long ofs = entry->offset - entry->delta->offset;
+ off_t ofs = entry->offset - entry->delta->offset;
unsigned pos = sizeof(header) - 1;
header[pos] = ofs & 127;
while (ofs >>= 7)
hdrlen += 20;
}
- use_packed_git(p);
- buf = (char *) p->pack_base
- + entry->in_pack_offset
- + entry->in_pack_header_size;
+ offset = entry->in_pack_offset + entry->in_pack_header_size;
datalen = find_packed_object_size(p, entry->in_pack_offset)
- entry->in_pack_header_size;
- if (!pack_to_stdout && check_inflate(buf, datalen, entry->size))
+ if (!pack_to_stdout && check_pack_inflate(p, &w_curs,
+ offset, datalen, entry->size))
die("corrupt delta in pack %s", sha1_to_hex(entry->sha1));
- sha1write(f, buf, datalen);
- unuse_packed_git(p);
+ copy_pack_data(f, p, &w_curs, offset, datalen);
+ unuse_pack(&w_curs);
reused++;
}
if (entry->delta)
return hdrlen + datalen;
}
-static unsigned long write_one(struct sha1file *f,
+static off_t write_one(struct sha1file *f,
struct object_entry *e,
- unsigned long offset)
+ off_t offset)
{
if (e->offset || e->preferred_base)
/* offset starts from header size and cannot be zero
static void write_pack_file(void)
{
- int i;
+ uint32_t i;
struct sha1file *f;
- unsigned long offset;
+ off_t offset;
struct pack_header hdr;
unsigned last_percent = 999;
int do_progress = progress;
f = sha1create("%s-%s.%s", base_name,
sha1_to_hex(object_list_sha1), "pack");
if (do_progress)
- fprintf(stderr, "Writing %d objects.\n", nr_result);
+ fprintf(stderr, "Writing %u objects.\n", nr_result);
hdr.hdr_signature = htonl(PACK_SIGNATURE);
hdr.hdr_version = htonl(PACK_VERSION);
fputc('\n', stderr);
done:
if (written != nr_result)
- die("wrote %d objects while expecting %d", written, nr_result);
+ die("wrote %u objects while expecting %u", written, nr_result);
sha1close(f, pack_file_sha1, 1);
}
static void write_index_file(void)
{
- int i;
+ uint32_t i;
struct sha1file *f = sha1create("%s-%s.%s", base_name,
sha1_to_hex(object_list_sha1), "idx");
struct object_entry **list = sorted_by_sha;
struct object_entry **last = list + nr_result;
- unsigned int array[256];
+ uint32_t array[256];
/*
* Write the first-level table (the list is sorted,
array[i] = htonl(next - sorted_by_sha);
list = next;
}
- sha1write(f, array, 256 * sizeof(int));
+ sha1write(f, array, 256 * 4);
/*
* Write the actual SHA1 entries..
list = sorted_by_sha;
for (i = 0; i < nr_result; i++) {
struct object_entry *entry = *list++;
- unsigned int offset = htonl(entry->offset);
+ uint32_t offset = htonl(entry->offset);
sha1write(f, &offset, 4);
sha1write(f, entry->sha1, 20);
}
static void rehash_objects(void)
{
- int i;
+ uint32_t i;
struct object_entry *oe;
object_ix_hashsz = nr_objects * 3;
static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclude)
{
- unsigned int idx = nr_objects;
+ uint32_t idx = nr_objects;
struct object_entry *entry;
struct packed_git *p;
- unsigned int found_offset = 0;
+ off_t found_offset = 0;
struct packed_git *found_pack = NULL;
int ix, status = 0;
if (!exclude) {
for (p = packed_git; p; p = p->next) {
- unsigned long offset = find_pack_entry_one(sha1, p);
+ off_t offset = find_pack_entry_one(sha1, p);
if (offset) {
if (incremental)
return 0;
@@ -651,9 +694,8 @@ static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclud
goto already_added;
if (idx >= nr_alloc) {
- unsigned int needed = (idx + 1024) * 3 / 2;
- objects = xrealloc(objects, needed * sizeof(*entry));
- nr_alloc = needed;
+ nr_alloc = (idx + 1024) * 3 / 2;
+ objects = xrealloc(objects, nr_alloc * sizeof(*entry));
}
entry = objects + idx;
nr_objects = idx + 1;
@@ -673,7 +715,7 @@ static int add_object_entry(const unsigned char *sha1, unsigned hash, int exclud
already_added:
if (progress_update) {
- fprintf(stderr, "Counting objects...%d\r", nr_objects);
+ fprintf(stderr, "Counting objects...%u\r", nr_objects);
progress_update = 0;
}
if (exclude)
struct pbase_tree_cache *ent, *nent;
void *data;
unsigned long size;
- char type[20];
+ enum object_type type;
int neigh;
int my_ix = pbase_tree_cache_ix(sha1);
int available_ix = -1;
/* Did not find one. Either we got a bogus request or
* we need to read and perhaps cache.
*/
- data = read_sha1_file(sha1, type, &size);
+ data = read_sha1_file(sha1, &type, &size);
if (!data)
return NULL;
- if (strcmp(type, tree_type)) {
+ if (type != OBJ_TREE) {
free(data);
return NULL;
}
while (tree_entry(tree,&entry)) {
unsigned long size;
- char type[20];
+ enum object_type type;
if (entry.pathlen != cmplen ||
memcmp(entry.path, name, cmplen) ||
!has_sha1_file(entry.sha1) ||
- sha1_object_info(entry.sha1, type, &size))
+ (type = sha1_object_info(entry.sha1, &size)) < 0)
continue;
if (name[cmplen] != '/') {
unsigned hash = name_hash(fullname);
add_object_entry(entry.sha1, hash, 1);
return;
}
- if (!strcmp(type, tree_type)) {
+ if (type == OBJ_TREE) {
struct tree_desc sub;
struct pbase_tree_cache *tree;
const char *down = name+cmplen+1;
static void check_object(struct object_entry *entry)
{
- char type[20];
-
if (entry->in_pack && !entry->preferred_base) {
struct packed_git *p = entry->in_pack;
- unsigned long left = p->pack_size - entry->in_pack_offset;
+ struct pack_window *w_curs = NULL;
unsigned long size, used;
+ unsigned int avail;
unsigned char *buf;
struct object_entry *base_entry = NULL;
- use_packed_git(p);
- buf = p->pack_base;
- buf += entry->in_pack_offset;
+ buf = use_pack(p, &w_curs, entry->in_pack_offset, &avail);
/* We want in_pack_type even if we do not reuse delta.
* There is no point not reusing non-delta representations.
*/
- used = unpack_object_header_gently(buf, left,
+ used = unpack_object_header_gently(buf, avail,
&entry->in_pack_type, &size);
- if (!used || left - used <= 20)
- die("corrupt pack for %s", sha1_to_hex(entry->sha1));
/* Check if it is delta, and the base is also an object
* we are going to pack. If so we will reuse the existing
*/
if (!no_reuse_delta) {
unsigned char c, *base_name;
- unsigned long ofs;
+ off_t ofs;
+ unsigned long used_0;
/* there is at least 20 bytes left in the pack */
switch (entry->in_pack_type) {
case OBJ_REF_DELTA:
- base_name = buf + used;
+ base_name = use_pack(p, &w_curs,
+ entry->in_pack_offset + used, NULL);
used += 20;
break;
case OBJ_OFS_DELTA:
- c = buf[used++];
+ buf = use_pack(p, &w_curs,
+ entry->in_pack_offset + used, NULL);
+ used_0 = 0;
+ c = buf[used_0++];
ofs = c & 127;
while (c & 128) {
ofs += 1;
if (!ofs || ofs & ~(~0UL >> 7))
die("delta base offset overflow in pack for %s",
sha1_to_hex(entry->sha1));
- c = buf[used++];
+ c = buf[used_0++];
ofs = (ofs << 7) + (c & 127);
}
if (ofs >= entry->in_pack_offset)
sha1_to_hex(entry->sha1));
ofs = entry->in_pack_offset - ofs;
base_name = find_packed_object_name(p, ofs);
+ used += used_0;
break;
default:
base_name = NULL;
if (base_name)
base_entry = locate_object_entry(base_name);
}
- unuse_packed_git(p);
+ unuse_pack(&w_curs);
entry->in_pack_header_size = used;
if (base_entry) {
/* Otherwise we would do the usual */
}
- if (sha1_object_info(entry->sha1, type, &entry->size))
+ entry->type = sha1_object_info(entry->sha1, &entry->size);
+ if (entry->type < 0)
die("unable to get type of object %s",
sha1_to_hex(entry->sha1));
-
- if (!strcmp(type, commit_type)) {
- entry->type = OBJ_COMMIT;
- } else if (!strcmp(type, tree_type)) {
- entry->type = OBJ_TREE;
- } else if (!strcmp(type, blob_type)) {
- entry->type = OBJ_BLOB;
- } else if (!strcmp(type, tag_type)) {
- entry->type = OBJ_TAG;
- } else
- die("unable to pack object %s of type %s",
- sha1_to_hex(entry->sha1), type);
}
static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
@@ -1046,7 +1078,7 @@ static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
static void get_object_details(void)
{
- int i;
+ uint32_t i;
struct object_entry *entry;
prepare_pack_ix();
static struct object_entry **create_sorted_list(entry_sort_t sort)
{
struct object_entry **list = xmalloc(nr_objects * sizeof(struct object_entry *));
- int i;
+ uint32_t i;
for (i = 0; i < nr_objects; i++)
list[i] = objects + i;
@@ -1102,7 +1134,7 @@ static int sha1_sort(const struct object_entry *a, const struct object_entry *b)
static struct object_entry **create_final_object_list(void)
{
struct object_entry **list;
- int i, j;
+ uint32_t i, j;
for (i = nr_result = 0; i < nr_objects; i++)
if (!objects[i].preferred_base)
struct object_entry *trg_entry = trg->entry;
struct object_entry *src_entry = src->entry;
unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;
- char type[10];
+ enum object_type type;
void *delta_buf;
/* Don't bother doing diffs between different types */
/* Load data if not already done */
if (!trg->data) {
- trg->data = read_sha1_file(trg_entry->sha1, type, &sz);
+ trg->data = read_sha1_file(trg_entry->sha1, &type, &sz);
if (sz != trg_size)
die("object %s inconsistent object length (%lu vs %lu)",
sha1_to_hex(trg_entry->sha1), sz, trg_size);
}
if (!src->data) {
- src->data = read_sha1_file(src_entry->sha1, type, &sz);
+ src->data = read_sha1_file(src_entry->sha1, &type, &sz);
if (sz != src_size)
die("object %s inconsistent object length (%lu vs %lu)",
sha1_to_hex(src_entry->sha1), sz, src_size);
static void find_deltas(struct object_entry **list, int window, int depth)
{
- int i, idx;
+ uint32_t i = nr_objects, idx = 0, processed = 0;
unsigned int array_size = window * sizeof(struct unpacked);
- struct unpacked *array = xmalloc(array_size);
- unsigned processed = 0;
+ struct unpacked *array;
unsigned last_percent = 999;
+ if (!nr_objects)
+ return;
+ array = xmalloc(array_size);
memset(array, 0, array_size);
- i = nr_objects;
- idx = 0;
if (progress)
- fprintf(stderr, "Deltifying %d objects.\n", nr_result);
+ fprintf(stderr, "Deltifying %u objects.\n", nr_result);
- while (--i >= 0) {
- struct object_entry *entry = list[i];
+ do {
+ struct object_entry *entry = list[--i];
struct unpacked *n = array + idx;
int j;
j = window;
while (--j > 0) {
- unsigned int other_idx = idx + j;
+ uint32_t other_idx = idx + j;
struct unpacked *m;
if (other_idx >= window)
other_idx -= window;
idx++;
if (idx >= window)
idx = 0;
- }
+ } while (i > 0);
if (progress)
fputc('\n', stderr);
}
if (progress)
- fprintf(stderr, "Reusing %d objects pack %s\n", nr_objects,
+ fprintf(stderr, "Reusing %u objects pack %s\n", nr_objects,
sha1_to_hex(sha1));
if (pack_to_stdout) {
struct object_entry **list;
int use_internal_rev_list = 0;
int thin = 0;
- int i;
- const char *rp_av[64];
+ uint32_t i;
+ const char **rp_av;
+ int rp_ac_alloc = 64;
int rp_ac;
+ rp_av = xcalloc(rp_ac_alloc, sizeof(*rp_av));
+
rp_av[0] = "pack-objects";
rp_av[1] = "--objects"; /* --thin will make it --objects-edge */
rp_ac = 2;
incremental = 1;
continue;
}
- if (!strncmp("--window=", arg, 9)) {
+ if (!prefixcmp(arg, "--window=")) {
char *end;
window = strtoul(arg+9, &end, 0);
if (!arg[9] || *end)
usage(pack_usage);
continue;
}
- if (!strncmp("--depth=", arg, 8)) {
+ if (!prefixcmp(arg, "--depth=")) {
char *end;
depth = strtoul(arg+8, &end, 0);
if (!arg[8] || *end)
continue;
}
if (!strcmp("--unpacked", arg) ||
- !strncmp("--unpacked=", arg, 11) ||
+ !prefixcmp(arg, "--unpacked=") ||
+ !strcmp("--reflog", arg) ||
!strcmp("--all", arg)) {
use_internal_rev_list = 1;
- if (ARRAY_SIZE(rp_av) - 1 <= rp_ac)
- die("too many internal rev-list options");
+ if (rp_ac >= rp_ac_alloc - 1) {
+ rp_ac_alloc = alloc_nr(rp_ac_alloc);
+ rp_av = xrealloc(rp_av,
+ rp_ac_alloc * sizeof(*rp_av));
+ }
rp_av[rp_ac++] = arg;
continue;
}
}
if (progress)
- fprintf(stderr, "Done counting %d objects.\n", nr_objects);
+ fprintf(stderr, "Done counting %u objects.\n", nr_objects);
sorted_by_sha = create_final_object_list();
if (non_empty && !nr_result)
return 0;
}
SHA1_Final(object_list_sha1, &ctx);
if (progress && (nr_objects != nr_result))
- fprintf(stderr, "Result has %d objects.\n", nr_result);
+ fprintf(stderr, "Result has %u objects.\n", nr_result);
if (reuse_cached_pack(object_list_sha1))
;
}
}
if (progress)
- fprintf(stderr, "Total %d (delta %d), reused %d (delta %d)\n",
+ fprintf(stderr, "Total %u (delta %u), reused %u (delta %u)\n",
written, written_delta, reused, reused_delta);
return 0;
}