![[tokkee]](http://tokkee.org/images/avatar.png){kind=avatar}
diff --git a/builtin-pack-objects.c b/builtin-pack-objects.c
index ccb25f6a9c298bd977c5f55a84dd9a20a1e54e23..12509faa777bb2903e98c79a98be151380911b87 100644 (file)
--- a/builtin-pack-objects.c
+++ b/builtin-pack-objects.c
#include "progress.h"
static const char pack_usage[] = "\
-git-pack-objects [{ -q | --progress | --all-progress }] [--max-pack-size=N] \n\
- [--local] [--incremental] [--window=N] [--depth=N] \n\
+git-pack-objects [{ -q | --progress | --all-progress }] \n\
+ [--max-pack-size=N] [--local] [--incremental] \n\
+ [--window=N] [--window-memory=N] [--depth=N] \n\
[--no-reuse-delta] [--no-reuse-object] [--delta-base-offset] \n\
[--non-empty] [--revs [--unpacked | --all]*] [--reflog] \n\
[--stdout | base-name] [<ref-list | <object-list]";
struct object_entry {
- unsigned char sha1[20];
- uint32_t crc32; /* crc of raw pack data for this object */
- off_t offset; /* offset into the final pack file */
+ struct pack_idx_entry idx;
unsigned long size; /* uncompressed size */
- unsigned int hash; /* name hint hash */
- unsigned int depth; /* delta depth */
struct packed_git *in_pack; /* already in pack */
off_t in_pack_offset;
struct object_entry *delta; /* delta base object */
*/
void *delta_data; /* cached delta (uncompressed) */
unsigned long delta_size; /* delta data size (uncompressed) */
+ unsigned int hash; /* name hint hash */
enum object_type type;
enum object_type in_pack_type; /* could be delta */
unsigned char in_pack_header_size;
static const char *pack_tmp_name, *idx_tmp_name;
static char tmpname[PATH_MAX];
static const char *base_name;
-static unsigned char pack_file_sha1[20];
static int progress = 1;
static int window = 10;
static uint32_t pack_size_limit;
static unsigned long max_delta_cache_size = 0;
static unsigned long cache_max_small_delta_size = 1000;
+static unsigned long window_memory_usage = 0;
+static unsigned long window_memory_limit = 0;
+
/*
* The object names in objects array are hashed with this hashtable,
* to help looking up the entry by object name.
@@ -246,15 +246,15 @@ static void *delta_against(void *buf, unsigned long size, struct object_entry *e
{
unsigned long othersize, delta_size;
enum object_type type;
- void *otherbuf = read_sha1_file(entry->delta->sha1, &type, &othersize);
+ void *otherbuf = read_sha1_file(entry->delta->idx.sha1, &type, &othersize);
void *delta_buf;
if (!otherbuf)
- die("unable to read %s", sha1_to_hex(entry->delta->sha1));
+ die("unable to read %s", sha1_to_hex(entry->delta->idx.sha1));
delta_buf = diff_delta(otherbuf, othersize,
buf, size, &delta_size, 0);
if (!delta_buf || delta_size != entry->delta_size)
- die("delta size changed");
+ die("delta size changed");
free(buf);
free(otherbuf);
return delta_buf;
/* yes if unlimited packfile */
!pack_size_limit ? 1 :
/* no if base written to previous pack */
- entry->delta->offset == (off_t)-1 ? 0 :
+ entry->delta->idx.offset == (off_t)-1 ? 0 :
/* otherwise double-check written to this
* pack, like we do below
*/
- entry->delta->offset ? 1 : 0;
+ entry->delta->idx.offset ? 1 : 0;
if (!pack_to_stdout)
crc32_begin(f);
unsigned long maxsize;
void *out;
if (!usable_delta) {
- buf = read_sha1_file(entry->sha1, &obj_type, &size);
+ buf = read_sha1_file(entry->idx.sha1, &obj_type, &size);
if (!buf)
- die("unable to read %s", sha1_to_hex(entry->sha1));
+ die("unable to read %s", sha1_to_hex(entry->idx.sha1));
} else if (entry->delta_data) {
size = entry->delta_size;
buf = entry->delta_data;
entry->delta_data = NULL;
- obj_type = (allow_ofs_delta && entry->delta->offset) ?
+ obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA;
} else {
- buf = read_sha1_file(entry->sha1, &type, &size);
+ buf = read_sha1_file(entry->idx.sha1, &type, &size);
if (!buf)
- die("unable to read %s", sha1_to_hex(entry->sha1));
+ die("unable to read %s", sha1_to_hex(entry->idx.sha1));
buf = delta_against(buf, size, entry);
size = entry->delta_size;
- obj_type = (allow_ofs_delta && entry->delta->offset) ?
+ obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA;
}
/* compress the data to store and put compressed length in datalen */
* encoding of the relative offset for the delta
* base from this object's position in the pack.
*/
- off_t ofs = entry->offset - entry->delta->offset;
+ off_t ofs = entry->idx.offset - entry->delta->idx.offset;
unsigned pos = sizeof(dheader) - 1;
dheader[pos] = ofs & 127;
while (ofs >>= 7)
return 0;
}
sha1write(f, header, hdrlen);
- sha1write(f, entry->delta->sha1, 20);
+ sha1write(f, entry->delta->idx.sha1, 20);
hdrlen += 20;
} else {
if (limit && hdrlen + datalen + 20 >= limit) {
off_t offset;
if (entry->delta) {
- obj_type = (allow_ofs_delta && entry->delta->offset) ?
+ obj_type = (allow_ofs_delta && entry->delta->idx.offset) ?
OBJ_OFS_DELTA : OBJ_REF_DELTA;
reused_delta++;
}
datalen = revidx[1].offset - offset;
if (!pack_to_stdout && p->index_version > 1 &&
check_pack_crc(p, &w_curs, offset, datalen, revidx->nr))
- die("bad packed object CRC for %s", sha1_to_hex(entry->sha1));
+ die("bad packed object CRC for %s", sha1_to_hex(entry->idx.sha1));
offset += entry->in_pack_header_size;
datalen -= entry->in_pack_header_size;
if (obj_type == OBJ_OFS_DELTA) {
- off_t ofs = entry->offset - entry->delta->offset;
+ off_t ofs = entry->idx.offset - entry->delta->idx.offset;
unsigned pos = sizeof(dheader) - 1;
dheader[pos] = ofs & 127;
while (ofs >>= 7)
if (limit && hdrlen + 20 + datalen + 20 >= limit)
return 0;
sha1write(f, header, hdrlen);
- sha1write(f, entry->delta->sha1, 20);
+ sha1write(f, entry->delta->idx.sha1, 20);
hdrlen += 20;
} else {
if (limit && hdrlen + datalen + 20 >= limit)
if (!pack_to_stdout && p->index_version == 1 &&
check_pack_inflate(p, &w_curs, offset, datalen, entry->size))
- die("corrupt packed object for %s", sha1_to_hex(entry->sha1));
+ die("corrupt packed object for %s", sha1_to_hex(entry->idx.sha1));
copy_pack_data(f, p, &w_curs, offset, datalen);
unuse_pack(&w_curs);
reused++;
written_delta++;
written++;
if (!pack_to_stdout)
- entry->crc32 = crc32_end(f);
+ entry->idx.crc32 = crc32_end(f);
return hdrlen + datalen;
}
unsigned long size;
/* offset is non zero if object is written already. */
- if (e->offset || e->preferred_base)
+ if (e->idx.offset || e->preferred_base)
return offset;
/* if we are deltified, write out base object first. */
return 0;
}
- e->offset = offset;
+ e->idx.offset = offset;
size = write_object(f, e, offset);
if (!size) {
- e->offset = 0;
+ e->idx.offset = 0;
return 0;
}
written_list[nr_written++] = e;
static int open_object_dir_tmp(const char *path)
{
snprintf(tmpname, sizeof(tmpname), "%s/%s", get_object_directory(), path);
- return mkstemp(tmpname);
+ return xmkstemp(tmpname);
}
-/* forward declarations for write_pack_file */
-static void write_index_file(off_t last_obj_offset, unsigned char *sha1);
+/* forward declaration for write_pack_file */
static int adjust_perm(const char *path, mode_t mode);
static void write_pack_file(void)
written_list = xmalloc(nr_objects * sizeof(struct object_entry *));
do {
+ unsigned char sha1[20];
+
if (pack_to_stdout) {
f = sha1fd(1, "<stdout>");
} else {
int fd = open_object_dir_tmp("tmp_pack_XXXXXX");
- if (fd < 0)
- die("unable to create %s: %s\n", tmpname, strerror(errno));
pack_tmp_name = xstrdup(tmpname);
f = sha1fd(fd, pack_tmp_name);
}
* If so, rewrite it like in fast-import
*/
if (pack_to_stdout || nr_written == nr_remaining) {
- sha1close(f, pack_file_sha1, 1);
+ sha1close(f, sha1, 1);
} else {
- sha1close(f, pack_file_sha1, 0);
- fixup_pack_header_footer(f->fd, pack_file_sha1, pack_tmp_name, nr_written);
+ sha1close(f, sha1, 0);
+ fixup_pack_header_footer(f->fd, sha1, pack_tmp_name, nr_written);
close(f->fd);
}
if (!pack_to_stdout) {
- unsigned char object_list_sha1[20];
mode_t mode = umask(0);
umask(mode);
mode = 0444 & ~mode;
- write_index_file(last_obj_offset, object_list_sha1);
+ idx_tmp_name = write_idx_file(NULL,
+ (struct pack_idx_entry **) written_list, nr_written, sha1);
snprintf(tmpname, sizeof(tmpname), "%s-%s.pack",
- base_name, sha1_to_hex(object_list_sha1));
+ base_name, sha1_to_hex(sha1));
if (adjust_perm(pack_tmp_name, mode))
die("unable to make temporary pack file readable: %s",
strerror(errno));
die("unable to rename temporary pack file: %s",
strerror(errno));
snprintf(tmpname, sizeof(tmpname), "%s-%s.idx",
- base_name, sha1_to_hex(object_list_sha1));
+ base_name, sha1_to_hex(sha1));
if (adjust_perm(idx_tmp_name, mode))
die("unable to make temporary index file readable: %s",
strerror(errno));
if (rename(idx_tmp_name, tmpname))
die("unable to rename temporary index file: %s",
strerror(errno));
- puts(sha1_to_hex(object_list_sha1));
+ puts(sha1_to_hex(sha1));
}
/* mark written objects as written to previous pack */
for (j = 0; j < nr_written; j++) {
- written_list[j]->offset = (off_t)-1;
+ written_list[j]->idx.offset = (off_t)-1;
}
nr_remaining -= nr_written;
} while (nr_remaining && i < nr_objects);
*/
for (j = 0; i < nr_objects; i++) {
struct object_entry *e = objects + i;
- j += !e->offset && !e->preferred_base;
+ j += !e->idx.offset && !e->preferred_base;
}
if (j)
die("wrote %u objects as expected but %u unwritten", written, j);
}
-static int sha1_sort(const void *_a, const void *_b)
-{
- const struct object_entry *a = *(struct object_entry **)_a;
- const struct object_entry *b = *(struct object_entry **)_b;
- return hashcmp(a->sha1, b->sha1);
-}
-
-static uint32_t index_default_version = 1;
-static uint32_t index_off32_limit = 0x7fffffff;
-
-static void write_index_file(off_t last_obj_offset, unsigned char *sha1)
-{
- struct sha1file *f;
- struct object_entry **sorted_by_sha, **list, **last;
- uint32_t array[256];
- uint32_t i, index_version;
- SHA_CTX ctx;
-
- int fd = open_object_dir_tmp("tmp_idx_XXXXXX");
- if (fd < 0)
- die("unable to create %s: %s\n", tmpname, strerror(errno));
- idx_tmp_name = xstrdup(tmpname);
- f = sha1fd(fd, idx_tmp_name);
-
- if (nr_written) {
- sorted_by_sha = written_list;
- qsort(sorted_by_sha, nr_written, sizeof(*sorted_by_sha), sha1_sort);
- list = sorted_by_sha;
- last = sorted_by_sha + nr_written;
- } else
- sorted_by_sha = list = last = NULL;
-
- /* if last object's offset is >= 2^31 we should use index V2 */
- index_version = (last_obj_offset >> 31) ? 2 : index_default_version;
-
- /* index versions 2 and above need a header */
- if (index_version >= 2) {
- struct pack_idx_header hdr;
- hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
- hdr.idx_version = htonl(index_version);
- sha1write(f, &hdr, sizeof(hdr));
- }
-
- /*
- * Write the first-level table (the list is sorted,
- * but we use a 256-entry lookup to be able to avoid
- * having to do eight extra binary search iterations).
- */
- for (i = 0; i < 256; i++) {
- struct object_entry **next = list;
- while (next < last) {
- struct object_entry *entry = *next;
- if (entry->sha1[0] != i)
- break;
- next++;
- }
- array[i] = htonl(next - sorted_by_sha);
- list = next;
- }
- sha1write(f, array, 256 * 4);
-
- /* Compute the SHA1 hash of sorted object names. */
- SHA1_Init(&ctx);
-
- /* Write the actual SHA1 entries. */
- list = sorted_by_sha;
- for (i = 0; i < nr_written; i++) {
- struct object_entry *entry = *list++;
- if (index_version < 2) {
- uint32_t offset = htonl(entry->offset);
- sha1write(f, &offset, 4);
- }
- sha1write(f, entry->sha1, 20);
- SHA1_Update(&ctx, entry->sha1, 20);
- }
-
- if (index_version >= 2) {
- unsigned int nr_large_offset = 0;
-
- /* write the crc32 table */
- list = sorted_by_sha;
- for (i = 0; i < nr_written; i++) {
- struct object_entry *entry = *list++;
- uint32_t crc32_val = htonl(entry->crc32);
- sha1write(f, &crc32_val, 4);
- }
-
- /* write the 32-bit offset table */
- list = sorted_by_sha;
- for (i = 0; i < nr_written; i++) {
- struct object_entry *entry = *list++;
- uint32_t offset = (entry->offset <= index_off32_limit) ?
- entry->offset : (0x80000000 | nr_large_offset++);
- offset = htonl(offset);
- sha1write(f, &offset, 4);
- }
-
- /* write the large offset table */
- list = sorted_by_sha;
- while (nr_large_offset) {
- struct object_entry *entry = *list++;
- uint64_t offset = entry->offset;
- if (offset > index_off32_limit) {
- uint32_t split[2];
- split[0] = htonl(offset >> 32);
- split[1] = htonl(offset & 0xffffffff);
- sha1write(f, split, 8);
- nr_large_offset--;
- }
- }
- }
-
- sha1write(f, pack_file_sha1, 20);
- sha1close(f, NULL, 1);
- SHA1_Final(sha1, &ctx);
-}
-
static int locate_object_entry_hash(const unsigned char *sha1)
{
int i;
memcpy(&ui, sha1, sizeof(unsigned int));
i = ui % object_ix_hashsz;
while (0 < object_ix[i]) {
- if (!hashcmp(sha1, objects[object_ix[i] - 1].sha1))
+ if (!hashcmp(sha1, objects[object_ix[i] - 1].idx.sha1))
return i;
if (++i == object_ix_hashsz)
i = 0;
object_ix = xrealloc(object_ix, sizeof(int) * object_ix_hashsz);
memset(object_ix, 0, sizeof(int) * object_ix_hashsz);
for (i = 0, oe = objects; i < nr_objects; i++, oe++) {
- int ix = locate_object_entry_hash(oe->sha1);
+ int ix = locate_object_entry_hash(oe->idx.sha1);
if (0 <= ix)
continue;
ix = -1 - ix;
entry = objects + nr_objects++;
memset(entry, 0, sizeof(*entry));
- hashcpy(entry->sha1, sha1);
+ hashcpy(entry->idx.sha1, sha1);
entry->hash = hash;
if (type)
entry->type = type;
int cmp;
while (tree_entry(tree,&entry)) {
+ if (S_ISGITLINK(entry.mode))
+ continue;
cmp = tree_entry_len(entry.path, entry.sha1) != cmplen ? 1 :
memcmp(name, entry.path, cmplen);
if (cmp > 0)
ofs += 1;
if (!ofs || MSB(ofs, 7))
die("delta base offset overflow in pack for %s",
- sha1_to_hex(entry->sha1));
+ sha1_to_hex(entry->idx.sha1));
c = buf[used_0++];
ofs = (ofs << 7) + (c & 127);
}
if (ofs >= entry->in_pack_offset)
die("delta base offset out of bound for %s",
- sha1_to_hex(entry->sha1));
+ sha1_to_hex(entry->idx.sha1));
ofs = entry->in_pack_offset - ofs;
if (!no_reuse_delta && !entry->preferred_base)
base_ref = find_packed_object_name(p, ofs);
unuse_pack(&w_curs);
}
- entry->type = sha1_object_info(entry->sha1, &entry->size);
+ entry->type = sha1_object_info(entry->idx.sha1, &entry->size);
if (entry->type < 0)
die("unable to get type of object %s",
- sha1_to_hex(entry->sha1));
+ sha1_to_hex(entry->idx.sha1));
}
static int pack_offset_sort(const void *_a, const void *_b)
/* avoid filesystem trashing with loose objects */
if (!a->in_pack && !b->in_pack)
- return hashcmp(a->sha1, b->sha1);
+ return hashcmp(a->idx.sha1, b->idx.sha1);
if (a->in_pack < b->in_pack)
return -1;
struct object_entry *entry;
void *data;
struct delta_index *index;
+ unsigned depth;
};
-static int delta_cacheable(struct unpacked *trg, struct unpacked *src,
- unsigned long src_size, unsigned long trg_size,
- unsigned long delta_size)
+static int delta_cacheable(unsigned long src_size, unsigned long trg_size,
+ unsigned long delta_size)
{
if (max_delta_cache_size && delta_cache_size + delta_size > max_delta_cache_size)
return 0;
struct object_entry *trg_entry = trg->entry;
struct object_entry *src_entry = src->entry;
unsigned long trg_size, src_size, delta_size, sizediff, max_size, sz;
+ unsigned ref_depth;
enum object_type type;
void *delta_buf;
return 0;
/* Let's not bust the allowed depth. */
- if (src_entry->depth >= max_depth)
+ if (src->depth >= max_depth)
return 0;
/* Now some size filtering heuristics. */
trg_size = trg_entry->size;
- max_size = trg_size/2 - 20;
- max_size = max_size * (max_depth - src_entry->depth) / max_depth;
+ if (!trg_entry->delta) {
+ max_size = trg_size/2 - 20;
+ ref_depth = 1;
+ } else {
+ max_size = trg_entry->delta_size;
+ ref_depth = trg->depth;
+ }
+ max_size = max_size * (max_depth - src->depth) /
+ (max_depth - ref_depth + 1);
if (max_size == 0)
return 0;
- if (trg_entry->delta && trg_entry->delta_size <= max_size)
- max_size = trg_entry->delta_size-1;
src_size = src_entry->size;
sizediff = src_size < trg_size ? trg_size - src_size : 0;
if (sizediff >= max_size)
return 0;
+ if (trg_size < src_size / 32)
+ return 0;
/* Load data if not already done */
if (!trg->data) {
- trg->data = read_sha1_file(trg_entry->sha1, &type, &sz);
+ trg->data = read_sha1_file(trg_entry->idx.sha1, &type, &sz);
+ if (!trg->data)
+ die("object %s cannot be read",
+ sha1_to_hex(trg_entry->idx.sha1));
if (sz != trg_size)
die("object %s inconsistent object length (%lu vs %lu)",
- sha1_to_hex(trg_entry->sha1), sz, trg_size);
+ sha1_to_hex(trg_entry->idx.sha1), sz, trg_size);
+ window_memory_usage += sz;
}
if (!src->data) {
- src->data = read_sha1_file(src_entry->sha1, &type, &sz);
+ src->data = read_sha1_file(src_entry->idx.sha1, &type, &sz);
+ if (!src->data)
+ die("object %s cannot be read",
+ sha1_to_hex(src_entry->idx.sha1));
if (sz != src_size)
die("object %s inconsistent object length (%lu vs %lu)",
- sha1_to_hex(src_entry->sha1), sz, src_size);
+ sha1_to_hex(src_entry->idx.sha1), sz, src_size);
+ window_memory_usage += sz;
}
if (!src->index) {
src->index = create_delta_index(src->data, src_size);
warning("suboptimal pack - out of memory");
return 0;
}
+ window_memory_usage += sizeof_delta_index(src->index);
}
delta_buf = create_delta(src->index, trg->data, trg_size, &delta_size, max_size);
if (!delta_buf)
return 0;
+ if (trg_entry->delta) {
+ /* Prefer only shallower same-sized deltas. */
+ if (delta_size == trg_entry->delta_size &&
+ src->depth + 1 >= trg->depth) {
+ free(delta_buf);
+ return 0;
+ }
+ }
+
+ trg_entry->delta = src_entry;
+ trg_entry->delta_size = delta_size;
+ trg->depth = src->depth + 1;
+
if (trg_entry->delta_data) {
delta_cache_size -= trg_entry->delta_size;
free(trg_entry->delta_data);
+ trg_entry->delta_data = NULL;
}
- trg_entry->delta_data = 0;
- trg_entry->delta = src_entry;
- trg_entry->delta_size = delta_size;
- trg_entry->depth = src_entry->depth + 1;
- if (delta_cacheable(src, trg, src_size, trg_size, delta_size)) {
+ if (delta_cacheable(src_size, trg_size, delta_size)) {
trg_entry->delta_data = xrealloc(delta_buf, delta_size);
delta_cache_size += trg_entry->delta_size;
} else
@@ -1518,9 +1429,23 @@ static unsigned int check_delta_limit(struct object_entry *me, unsigned int n)
return m;
}
+static void free_unpacked(struct unpacked *n)
+{
+ window_memory_usage -= sizeof_delta_index(n->index);
+ free_delta_index(n->index);
+ n->index = NULL;
+ if (n->data) {
+ free(n->data);
+ n->data = NULL;
+ window_memory_usage -= n->entry->size;
+ }
+ n->entry = NULL;
+ n->depth = 0;
+}
+
static void find_deltas(struct object_entry **list, int window, int depth)
{
- uint32_t i = nr_objects, idx = 0, processed = 0;
+ uint32_t i = nr_objects, idx = 0, count = 0, processed = 0;
unsigned int array_size = window * sizeof(struct unpacked);
struct unpacked *array;
int max_depth;
if (entry->no_try_delta)
continue;
- free_delta_index(n->index);
- n->index = NULL;
- free(n->data);
- n->data = NULL;
+ free_unpacked(n);
n->entry = entry;
+ while (window_memory_limit &&
+ window_memory_usage > window_memory_limit &&
+ count > 1) {
+ uint32_t tail = (idx + window - count) % window;
+ free_unpacked(array + tail);
+ count--;
+ }
+
/*
* If the current object is at pack edge, take the depth the
* objects that depend on the current object into account
* depth, leaving it in the window is pointless. we
* should evict it first.
*/
- if (entry->delta && depth <= entry->depth)
+ if (entry->delta && depth <= n->depth)
continue;
next:
idx++;
+ if (count + 1 < window)
+ count++;
if (idx >= window)
idx = 0;
} while (i > 0);
window = git_config_int(k, v);
return 0;
}
- if(!strcmp(k, "pack.depth")) {
+ if (!strcmp(k, "pack.windowmemory")) {
+ window_memory_limit = git_config_ulong(k, v);
+ return 0;
+ }
+ if (!strcmp(k, "pack.depth")) {
depth = git_config_int(k, v);
return 0;
}
usage(pack_usage);
continue;
}
+ if (!prefixcmp(arg, "--window-memory=")) {
+ if (!git_parse_ulong(arg+16, &window_memory_limit))
+ usage(pack_usage);
+ continue;
+ }
if (!prefixcmp(arg, "--depth=")) {
char *end;
depth = strtoul(arg+8, &end, 0);
}
if (!prefixcmp(arg, "--index-version=")) {
char *c;
- index_default_version = strtoul(arg + 16, &c, 10);
- if (index_default_version > 2)
+ pack_idx_default_version = strtoul(arg + 16, &c, 10);
+ if (pack_idx_default_version > 2)
die("bad %s", arg);
if (*c == ',')
- index_off32_limit = strtoul(c+1, &c, 0);
- if (*c || index_off32_limit & 0x80000000)
+ pack_idx_off32_limit = strtoul(c+1, &c, 0);
+ if (*c || pack_idx_off32_limit & 0x80000000)
die("bad %s", arg);
continue;
}