X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=index-pack.c;h=58c4a9c41dd7a05b86d40e6eeee33ba0a3fb6c4f;hb=cfabd6eee1745cfec58cfcb794ce8847e43b888a;hp=d33f723365719bef2370428a691832243aa7cb37;hpb=ee5743ce191d4f39f976e76fe4c12bf8fc67a590;p=git.git diff --git a/index-pack.c b/index-pack.c index d33f72336..58c4a9c41 100644 --- a/index-pack.c +++ b/index-pack.c @@ -6,6 +6,7 @@ #include "commit.h" #include "tag.h" #include "tree.h" +#include "progress.h" static const char index_pack_usage[] = "git-index-pack [-v] [-o ] [{ ---keep | --keep= }] { | --stdin [--fix-thin] [] }"; @@ -47,40 +48,7 @@ static int nr_resolved_deltas; static int from_stdin; static int verbose; -static volatile sig_atomic_t progress_update; - -static void progress_interval(int signum) -{ - progress_update = 1; -} - -static void setup_progress_signal(void) -{ - struct sigaction sa; - struct itimerval v; - - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = progress_interval; - sigemptyset(&sa.sa_mask); - sa.sa_flags = SA_RESTART; - sigaction(SIGALRM, &sa, NULL); - - v.it_interval.tv_sec = 1; - v.it_interval.tv_usec = 0; - v.it_value = v.it_interval; - setitimer(ITIMER_REAL, &v, NULL); - -} - -static unsigned display_progress(unsigned n, unsigned total, unsigned last_pc) -{ - unsigned percent = n * 100 / total; - if (percent != last_pc || progress_update) { - fprintf(stderr, "%4u%% (%u/%u) done\r", percent, n, total); - progress_update = 0; - } - return percent; -} +static struct progress progress; /* We always read in 4kB chunks. */ static unsigned char input_buffer[4096]; @@ -114,7 +82,7 @@ static void *fill(int min) die("cannot fill %d bytes", min); flush(); do { - int ret = xread(input_fd, input_buffer + input_len, + ssize_t ret = xread(input_fd, input_buffer + input_len, sizeof(input_buffer) - input_len); if (ret <= 0) { if (!ret) @@ -428,7 +396,7 @@ static int compare_delta_entry(const void *a, const void *b) /* Parse all objects and return the pack content SHA1 hash */ static void parse_pack_objects(unsigned char *sha1) { - int i, percent = -1; + int i; struct delta_entry *delta = deltas; void *data; struct stat st; @@ -440,7 +408,7 @@ static void parse_pack_objects(unsigned char *sha1) * - remember base (SHA1 or offset) for all deltas. */ if (verbose) - fprintf(stderr, "Indexing %d objects.\n", nr_objects); + start_progress(&progress, "Indexing %u objects...", "", nr_objects); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; data = unpack_raw_entry(obj, &delta->base); @@ -453,11 +421,11 @@ static void parse_pack_objects(unsigned char *sha1) sha1_object(data, obj->size, obj->type, obj->sha1); free(data); if (verbose) - percent = display_progress(i+1, nr_objects, percent); + display_progress(&progress, i+1); } objects[i].offset = consumed_bytes; if (verbose) - fputc('\n', stderr); + stop_progress(&progress); /* Check pack integrity */ flush(); @@ -489,7 +457,7 @@ static void parse_pack_objects(unsigned char *sha1) * for some more deltas. */ if (verbose) - fprintf(stderr, "Resolving %d deltas.\n", nr_deltas); + start_progress(&progress, "Resolving %u deltas...", "", nr_deltas); for (i = 0; i < nr_objects; i++) { struct object_entry *obj = &objects[i]; union delta_base base; @@ -521,11 +489,8 @@ static void parse_pack_objects(unsigned char *sha1) } free(data); if (verbose) - percent = display_progress(nr_resolved_deltas, - nr_deltas, percent); + display_progress(&progress, nr_resolved_deltas); } - if (verbose && nr_resolved_deltas == nr_deltas) - fputc('\n', stderr); } static int write_compressed(int fd, void *in, unsigned int size, uint32_t *obj_crc) @@ -587,7 +552,7 @@ static int delta_pos_compare(const void *_a, const void *_b) static void fix_unresolved_deltas(int nr_unresolved) { struct delta_entry **sorted_by_pos; - int i, n = 0, percent = -1; + int i, n = 0; /* * Since many unresolved deltas may well be themselves base objects @@ -632,44 +597,13 @@ static void fix_unresolved_deltas(int nr_unresolved) append_obj_to_pack(d->base.sha1, data, size, type); free(data); if (verbose) - percent = display_progress(nr_resolved_deltas, - nr_deltas, percent); + display_progress(&progress, nr_resolved_deltas); } free(sorted_by_pos); - if (verbose) - fputc('\n', stderr); } -static void readjust_pack_header_and_sha1(unsigned char *sha1) -{ - struct pack_header hdr; - SHA_CTX ctx; - int size; - - /* Rewrite pack header with updated object number */ - if (lseek(output_fd, 0, SEEK_SET) != 0) - die("cannot seek back: %s", strerror(errno)); - if (read_in_full(output_fd, &hdr, sizeof(hdr)) != sizeof(hdr)) - die("cannot read pack header back: %s", strerror(errno)); - hdr.hdr_entries = htonl(nr_objects); - if (lseek(output_fd, 0, SEEK_SET) != 0) - die("cannot seek back: %s", strerror(errno)); - write_or_die(output_fd, &hdr, sizeof(hdr)); - if (lseek(output_fd, 0, SEEK_SET) != 0) - die("cannot seek back: %s", strerror(errno)); - - /* Recompute and store the new pack's SHA1 */ - SHA1_Init(&ctx); - do { - unsigned char *buf[4096]; - size = xread(output_fd, buf, sizeof(buf)); - if (size < 0) - die("cannot read pack data back: %s", strerror(errno)); - SHA1_Update(&ctx, buf, size); - } while (size > 0); - SHA1_Final(sha1, &ctx); - write_or_die(output_fd, sha1, 20); -} +static uint32_t index_default_version = 1; +static uint32_t index_off32_limit = 0x7fffffff; static int sha1_compare(const void *_a, const void *_b) { @@ -686,9 +620,10 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1) { struct sha1file *f; struct object_entry **sorted_by_sha, **list, **last; - unsigned int array[256]; + uint32_t array[256]; int i, fd; SHA_CTX ctx; + uint32_t index_version; if (nr_objects) { sorted_by_sha = @@ -699,7 +634,6 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1) sorted_by_sha[i] = &objects[i]; qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]), sha1_compare); - } else sorted_by_sha = list = last = NULL; @@ -718,6 +652,17 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1) die("unable to create %s: %s", index_name, strerror(errno)); f = sha1fd(fd, index_name); + /* if last object's offset is >= 2^31 we should use index V2 */ + index_version = (objects[nr_objects-1].offset >> 31) ? 2 : index_default_version; + + /* index versions 2 and above need a header */ + if (index_version >= 2) { + struct pack_idx_header hdr; + hdr.idx_signature = htonl(PACK_IDX_SIGNATURE); + hdr.idx_version = htonl(index_version); + sha1write(f, &hdr, sizeof(hdr)); + } + /* * Write the first-level table (the list is sorted, * but we use a 256-entry lookup to be able to avoid @@ -734,24 +679,61 @@ static const char *write_index_file(const char *index_name, unsigned char *sha1) array[i] = htonl(next - sorted_by_sha); list = next; } - sha1write(f, array, 256 * sizeof(int)); + sha1write(f, array, 256 * 4); - /* recompute the SHA1 hash of sorted object names. - * currently pack-objects does not do this, but that - * can be fixed. - */ + /* compute the SHA1 hash of sorted object names. */ SHA1_Init(&ctx); + /* * Write the actual SHA1 entries.. */ list = sorted_by_sha; for (i = 0; i < nr_objects; i++) { struct object_entry *obj = *list++; - unsigned int offset = htonl(obj->offset); - sha1write(f, &offset, 4); + if (index_version < 2) { + uint32_t offset = htonl(obj->offset); + sha1write(f, &offset, 4); + } sha1write(f, obj->sha1, 20); SHA1_Update(&ctx, obj->sha1, 20); } + + if (index_version >= 2) { + unsigned int nr_large_offset = 0; + + /* write the crc32 table */ + list = sorted_by_sha; + for (i = 0; i < nr_objects; i++) { + struct object_entry *obj = *list++; + uint32_t crc32_val = htonl(obj->crc32); + sha1write(f, &crc32_val, 4); + } + + /* write the 32-bit offset table */ + list = sorted_by_sha; + for (i = 0; i < nr_objects; i++) { + struct object_entry *obj = *list++; + uint32_t offset = (obj->offset <= index_off32_limit) ? + obj->offset : (0x80000000 | nr_large_offset++); + offset = htonl(offset); + sha1write(f, &offset, 4); + } + + /* write the large offset table */ + list = sorted_by_sha; + while (nr_large_offset) { + struct object_entry *obj = *list++; + uint64_t offset = obj->offset; + if (offset > index_off32_limit) { + uint32_t split[2]; + split[0] = htonl(offset >> 32); + split[1] = htonl(offset & 0xffffffff); + sha1write(f, split, 8); + nr_large_offset--; + } + } + } + sha1write(f, sha1, 20); sha1close(f, NULL, 1); free(sorted_by_sha); @@ -881,6 +863,15 @@ int main(int argc, char **argv) if (index_name || (i+1) >= argc) usage(index_pack_usage); index_name = argv[++i]; + } else if (!prefixcmp(arg, "--index-version=")) { + char *c; + index_default_version = strtoul(arg + 16, &c, 10); + if (index_default_version > 2) + die("bad %s", arg); + if (*c == ',') + index_off32_limit = strtoul(c+1, &c, 0); + if (*c || index_off32_limit & 0x80000000) + die("bad %s", arg); } else usage(index_pack_usage); continue; @@ -920,10 +911,13 @@ int main(int argc, char **argv) parse_pack_header(); objects = xmalloc((nr_objects + 1) * sizeof(struct object_entry)); deltas = xmalloc(nr_objects * sizeof(struct delta_entry)); - if (verbose) - setup_progress_signal(); parse_pack_objects(sha1); - if (nr_deltas != nr_resolved_deltas) { + if (nr_deltas == nr_resolved_deltas) { + if (verbose) + stop_progress(&progress); + /* Flush remaining pack final 20-byte SHA1. */ + flush(); + } else { if (fix_thin_pack) { int nr_unresolved = nr_deltas - nr_resolved_deltas; int nr_objects_initial = nr_objects; @@ -933,17 +927,17 @@ int main(int argc, char **argv) (nr_objects + nr_unresolved + 1) * sizeof(*objects)); fix_unresolved_deltas(nr_unresolved); - if (verbose) + if (verbose) { + stop_progress(&progress); fprintf(stderr, "%d objects were added to complete this thin pack.\n", nr_objects - nr_objects_initial); - readjust_pack_header_and_sha1(sha1); + } + fixup_pack_header_footer(output_fd, sha1, + curr_pack, nr_objects); } if (nr_deltas != nr_resolved_deltas) die("pack has %d unresolved deltas", nr_deltas - nr_resolved_deltas); - } else { - /* Flush remaining pack final 20-byte SHA1. */ - flush(); } free(deltas); curr_index = write_index_file(index_name, sha1);