X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=index-pack.c;h=3c99a1fce97c387ec1dfc76649699a8095e76d81;hb=1eaa541f5fbda3fc30085da4f75e99dfae4ec9b3;hp=58c4a9c41dd7a05b86d40e6eeee33ba0a3fb6c4f;hpb=31c74ca67162674e3ff8fcc294b75881c3e7cc15;p=git.git

diff --git a/index-pack.c b/index-pack.c
index 58c4a9c41..9c0c27813 100644
--- a/index-pack.c
+++ b/index-pack.c
@@ -7,19 +7,18 @@
 #include "tag.h"
 #include "tree.h"
 #include "progress.h"
+#include "fsck.h"
 
 static const char index_pack_usage[] =
-"git-index-pack [-v] [-o <index-file>] [{ ---keep | --keep=<msg> }] { <pack-file> | --stdin [--fix-thin] [<pack-file>] }";
+"git-index-pack [-v] [-o <index-file>] [{ ---keep | --keep=<msg> }] [--strict] { <pack-file> | --stdin [--fix-thin] [<pack-file>] }";
 
 struct object_entry
 {
-	off_t offset;
+	struct pack_idx_entry idx;
 	unsigned long size;
 	unsigned int hdr_size;
-	uint32_t crc32;
 	enum object_type type;
 	enum object_type real_type;
-	unsigned char sha1[20];
 };
 
 union delta_base {
@@ -33,6 +32,9 @@ union delta_base {
  */
 #define UNION_BASE_SZ	20
 
+#define FLAG_LINK (1u<<20)
+#define FLAG_CHECKED (1u<<21)
+
 struct delta_entry
 {
 	union delta_base base;
@@ -46,9 +48,10 @@ static int nr_deltas;
 static int nr_resolved_deltas;
 
 static int from_stdin;
+static int strict;
 static int verbose;
 
-static struct progress progress;
+static struct progress *progress;
 
 /* We always read in 4kB chunks. */
 static unsigned char input_buffer[4096];
@@ -58,6 +61,48 @@ static SHA_CTX input_ctx;
 static uint32_t input_crc32;
 static int input_fd, output_fd, pack_fd;
 
+static int mark_link(struct object *obj, int type, void *data)
+{
+	if (!obj)
+		return -1;
+
+	if (type != OBJ_ANY && obj->type != type)
+		die("object type mismatch at %s", sha1_to_hex(obj->sha1));
+
+	obj->flags |= FLAG_LINK;
+	return 0;
+}
+
+/* The content of each linked object must have been checked
+   or it must be already present in the object database */
+static void check_object(struct object *obj)
+{
+	if (!obj)
+		return;
+
+	if (!(obj->flags & FLAG_LINK))
+		return;
+
+	if (!(obj->flags & FLAG_CHECKED)) {
+		unsigned long size;
+		int type = sha1_object_info(obj->sha1, &size);
+		if (type != obj->type || type <= 0)
+			die("object of unexpected type");
+		obj->flags |= FLAG_CHECKED;
+		return;
+	}
+}
+
+static void check_objects(void)
+{
+	unsigned i, max;
+
+	max = get_max_object_index();
+	for (i = 0; i < max; i++)
+		check_object(get_indexed_object(i));
+}
+
+
 /* Discard current buffer used content. */
 static void flush(void)
 {
@@ -90,6 +135,8 @@ static void *fill(int min)
 			die("read error on input: %s", strerror(errno));
 		}
 		input_len += ret;
+		if (from_stdin)
+			display_throughput(progress, consumed_bytes + input_len);
 	} while (input_len < min);
 	return input_buffer;
 }
@@ -108,7 +155,7 @@ static void use(int bytes)
 	consumed_bytes += bytes;
 }
 
-static const char *open_pack_file(const char *pack_name)
+static char *open_pack_file(char *pack_name)
 {
 	if (from_stdin) {
 		input_fd = 0;
@@ -116,7 +163,7 @@ static const char *open_pack_file(const char *pack_name)
 			static char tmpfile[PATH_MAX];
 			snprintf(tmpfile, sizeof(tmpfile),
 				 "%s/tmp_pack_XXXXXX", get_object_directory());
-			output_fd = mkstemp(tmpfile);
+			output_fd = xmkstemp(tmpfile);
 			pack_name = xstrdup(tmpfile);
 		} else
 			output_fd = open(pack_name, O_CREAT|O_EXCL|O_RDWR, 0600);
@@ -197,7 +244,7 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
 	unsigned shift;
 	void *data;
 
-	obj->offset = consumed_bytes;
+	obj->idx.offset = consumed_bytes;
 	input_crc32 = crc32(0, Z_NULL, 0);
 
 	p = fill(1);
@@ -229,15 +276,15 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
 		while (c & 128) {
 			base_offset += 1;
 			if (!base_offset || MSB(base_offset, 7))
-				bad_object(obj->offset, "offset value overflow for delta base object");
+				bad_object(obj->idx.offset, "offset value overflow for delta base object");
 			p = fill(1);
 			c = *p;
 			use(1);
 			base_offset = (base_offset << 7) + (c & 127);
 		}
-		delta_base->offset = obj->offset - base_offset;
-		if (delta_base->offset >= obj->offset)
-			bad_object(obj->offset, "delta base offset is out of bound");
+		delta_base->offset = obj->idx.offset - base_offset;
+		if (delta_base->offset >= obj->idx.offset)
+			bad_object(obj->idx.offset, "delta base offset is out of bound");
 		break;
 	case OBJ_COMMIT:
 	case OBJ_TREE:
@@ -245,19 +292,19 @@ static void *unpack_raw_entry(struct object_entry *obj, union delta_base *delta_
 	case OBJ_TAG:
 		break;
 	default:
-		bad_object(obj->offset, "unknown object type %d", obj->type);
+		bad_object(obj->idx.offset, "unknown object type %d", obj->type);
 	}
-	obj->hdr_size = consumed_bytes - obj->offset;
+	obj->hdr_size = consumed_bytes - obj->idx.offset;
 
-	data = unpack_entry_data(obj->offset, obj->size);
-	obj->crc32 = input_crc32;
+	data = unpack_entry_data(obj->idx.offset, obj->size);
+	obj->idx.crc32 = input_crc32;
 	return data;
 }
 
 static void *get_data_from_pack(struct object_entry *obj)
 {
-	unsigned long from = obj[0].offset + obj[0].hdr_size;
-	unsigned long len = obj[1].offset - from;
+	off_t from = obj[0].idx.offset + obj[0].hdr_size;
+	unsigned long len = obj[1].idx.offset - from;
 	unsigned long rdy = 0;
 	unsigned char *src, *data;
 	z_stream stream;
@@ -341,6 +388,41 @@ static void sha1_object(const void *data, unsigned long size,
 			die("SHA1 COLLISION FOUND WITH %s !", sha1_to_hex(sha1));
 		free(has_data);
 	}
+	if (strict) {
+		if (type == OBJ_BLOB) {
+			struct blob *blob = lookup_blob(sha1);
+			if (blob)
+				blob->object.flags |= FLAG_CHECKED;
+			else
+				die("invalid blob object %s", sha1_to_hex(sha1));
+		} else {
+			struct object *obj;
+			int eaten;
+			void *buf = (void *) data;
+
+			/*
+			 * we do not need to free the memory here, as the
+			 * buf is deleted by the caller.
+			 */
+			obj = parse_object_buffer(sha1, type, size, buf, &eaten);
+			if (!obj)
+				die("invalid %s", typename(type));
+			if (fsck_object(obj, 1, fsck_error_function))
+				die("Error in object");
+			if (fsck_walk(obj, mark_link, 0))
+				die("Not all child objects of %s are reachable", sha1_to_hex(obj->sha1));
+
+			if (obj->type == OBJ_TREE) {
+				struct tree *item = (struct tree *) obj;
+				item->buffer = NULL;
+			}
+			if (obj->type == OBJ_COMMIT) {
+				struct commit *commit = (struct commit *) obj;
+				commit->buffer = NULL;
+			}
+			obj->flags |= FLAG_CHECKED;
+		}
+	}
 }
 
 static void resolve_delta(struct object_entry *delta_obj, void *base_data,
@@ -360,11 +442,11 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
 			     &result_size);
 	free(delta_data);
 	if (!result)
-		bad_object(delta_obj->offset, "failed to apply delta");
-	sha1_object(result, result_size, type, delta_obj->sha1);
+		bad_object(delta_obj->idx.offset, "failed to apply delta");
+	sha1_object(result, result_size, type, delta_obj->idx.sha1);
 	nr_resolved_deltas++;
 
-	hashcpy(delta_base.sha1, delta_obj->sha1);
+	hashcpy(delta_base.sha1, delta_obj->idx.sha1);
 	if (!find_delta_children(&delta_base, &first, &last)) {
 		for (j = first; j <= last; j++) {
 			struct object_entry *child = objects + deltas[j].obj_no;
@@ -374,7 +456,7 @@ static void resolve_delta(struct object_entry *delta_obj, void *base_data,
 	}
 
 	memset(&delta_base, 0, sizeof(delta_base));
-	delta_base.offset = delta_obj->offset;
+	delta_base.offset = delta_obj->idx.offset;
 	if (!find_delta_children(&delta_base, &first, &last)) {
 		for (j = first; j <= last; j++) {
 			struct object_entry *child = objects + deltas[j].obj_no;
@@ -408,7 +490,9 @@ static void parse_pack_objects(unsigned char *sha1)
 	 * - remember base (SHA1 or offset) for all deltas.
 	 */
 	if (verbose)
-		start_progress(&progress, "Indexing %u objects...", "", nr_objects);
+		progress = start_progress(
+				from_stdin ? "Receiving objects" : "Indexing objects",
+				nr_objects);
 	for (i = 0; i < nr_objects; i++) {
 		struct object_entry *obj = &objects[i];
 		data = unpack_raw_entry(obj, &delta->base);
@@ -418,14 +502,12 @@ static void parse_pack_objects(unsigned char *sha1)
 			delta->obj_no = i;
 			delta++;
 		} else
-			sha1_object(data, obj->size, obj->type, obj->sha1);
+			sha1_object(data, obj->size, obj->type, obj->idx.sha1);
 		free(data);
-		if (verbose)
-			display_progress(&progress, i+1);
+		display_progress(progress, i+1);
 	}
-	objects[i].offset = consumed_bytes;
-	if (verbose)
-		stop_progress(&progress);
+	objects[i].idx.offset = consumed_bytes;
+	stop_progress(&progress);
 
 	/* Check pack integrity */
 	flush();
@@ -457,7 +539,7 @@ static void parse_pack_objects(unsigned char *sha1)
 	 *   for some more deltas.
 	 */
 	if (verbose)
-		start_progress(&progress, "Resolving %u deltas...", "", nr_deltas);
+		progress = start_progress("Resolving deltas", nr_deltas);
 	for (i = 0; i < nr_objects; i++) {
 		struct object_entry *obj = &objects[i];
 		union delta_base base;
@@ -465,10 +547,10 @@ static void parse_pack_objects(unsigned char *sha1)
 
 		if (obj->type == OBJ_REF_DELTA || obj->type == OBJ_OFS_DELTA)
 			continue;
-		hashcpy(base.sha1, obj->sha1);
+		hashcpy(base.sha1, obj->idx.sha1);
 		ref = !find_delta_children(&base, &ref_first, &ref_last);
 		memset(&base, 0, sizeof(base));
-		base.offset = obj->offset;
+		base.offset = obj->idx.offset;
 		ofs = !find_delta_children(&base, &ofs_first, &ofs_last);
 		if (!ref && !ofs)
 			continue;
@@ -488,8 +570,7 @@ static void parse_pack_objects(unsigned char *sha1)
 						      obj->size, obj->type);
 			}
 		free(data);
-		if (verbose)
-			display_progress(&progress, nr_resolved_deltas);
+		display_progress(progress, nr_resolved_deltas);
 	}
 }
 
@@ -535,11 +616,11 @@ static void append_obj_to_pack(const unsigned char *sha1, void *buf,
 	}
 	header[n++] = c;
 	write_or_die(output_fd, header, n);
-	obj[0].crc32 = crc32(0, Z_NULL, 0);
-	obj[0].crc32 = crc32(obj[0].crc32, header, n);
-	obj[1].offset = obj[0].offset + n;
-	obj[1].offset += write_compressed(output_fd, buf, size, &obj[0].crc32);
-	hashcpy(obj->sha1, sha1);
+	obj[0].idx.crc32 = crc32(0, Z_NULL, 0);
+	obj[0].idx.crc32 = crc32(obj[0].idx.crc32, header, n);
+	obj[1].idx.offset = obj[0].idx.offset + n;
+	obj[1].idx.offset += write_compressed(output_fd, buf, size, &obj[0].idx.crc32);
+	hashcpy(obj->idx.sha1, sha1);
 }
 
 static int delta_pos_compare(const void *_a, const void *_b)
@@ -596,151 +677,11 @@ static void fix_unresolved_deltas(int nr_unresolved)
 			die("local object %s is corrupt", sha1_to_hex(d->base.sha1));
 		append_obj_to_pack(d->base.sha1, data, size, type);
 		free(data);
-		if (verbose)
-			display_progress(&progress, nr_resolved_deltas);
+		display_progress(progress, nr_resolved_deltas);
 	}
 	free(sorted_by_pos);
 }
 
-static uint32_t index_default_version = 1;
-static uint32_t index_off32_limit = 0x7fffffff;
-
-static int sha1_compare(const void *_a, const void *_b)
-{
-	struct object_entry *a = *(struct object_entry **)_a;
-	struct object_entry *b = *(struct object_entry **)_b;
-	return hashcmp(a->sha1, b->sha1);
-}
-
-/*
- * On entry *sha1 contains the pack content SHA1 hash, on exit it is
- * the SHA1 hash of sorted object names.
- */
-static const char *write_index_file(const char *index_name, unsigned char *sha1)
-{
-	struct sha1file *f;
-	struct object_entry **sorted_by_sha, **list, **last;
-	uint32_t array[256];
-	int i, fd;
-	SHA_CTX ctx;
-	uint32_t index_version;
-
-	if (nr_objects) {
-		sorted_by_sha =
-			xcalloc(nr_objects, sizeof(struct object_entry *));
-		list = sorted_by_sha;
-		last = sorted_by_sha + nr_objects;
-		for (i = 0; i < nr_objects; ++i)
-			sorted_by_sha[i] = &objects[i];
-		qsort(sorted_by_sha, nr_objects, sizeof(sorted_by_sha[0]),
-		      sha1_compare);
-	}
-	else
-		sorted_by_sha = list = last = NULL;
-
-	if (!index_name) {
-		static char tmpfile[PATH_MAX];
-		snprintf(tmpfile, sizeof(tmpfile),
-			 "%s/tmp_idx_XXXXXX", get_object_directory());
-		fd = mkstemp(tmpfile);
-		index_name = xstrdup(tmpfile);
-	} else {
-		unlink(index_name);
-		fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600);
-	}
-	if (fd < 0)
-		die("unable to create %s: %s", index_name, strerror(errno));
-	f = sha1fd(fd, index_name);
-
-	/* if last object's offset is >= 2^31 we should use index V2 */
-	index_version = (objects[nr_objects-1].offset >> 31) ? 2 : index_default_version;
-
-	/* index versions 2 and above need a header */
-	if (index_version >= 2) {
-		struct pack_idx_header hdr;
-		hdr.idx_signature = htonl(PACK_IDX_SIGNATURE);
-		hdr.idx_version = htonl(index_version);
-		sha1write(f, &hdr, sizeof(hdr));
-	}
-
-	/*
-	 * Write the first-level table (the list is sorted,
-	 * but we use a 256-entry lookup to be able to avoid
-	 * having to do eight extra binary search iterations).
-	 */
-	for (i = 0; i < 256; i++) {
-		struct object_entry **next = list;
-		while (next < last) {
-			struct object_entry *obj = *next;
-			if (obj->sha1[0] != i)
-				break;
-			next++;
-		}
-		array[i] = htonl(next - sorted_by_sha);
-		list = next;
-	}
-	sha1write(f, array, 256 * 4);
-
-	/* compute the SHA1 hash of sorted object names. */
-	SHA1_Init(&ctx);
-
-	/*
-	 * Write the actual SHA1 entries..
-	 */
-	list = sorted_by_sha;
-	for (i = 0; i < nr_objects; i++) {
-		struct object_entry *obj = *list++;
-		if (index_version < 2) {
-			uint32_t offset = htonl(obj->offset);
-			sha1write(f, &offset, 4);
-		}
-		sha1write(f, obj->sha1, 20);
-		SHA1_Update(&ctx, obj->sha1, 20);
-	}
-
-	if (index_version >= 2) {
-		unsigned int nr_large_offset = 0;
-
-		/* write the crc32 table */
-		list = sorted_by_sha;
-		for (i = 0; i < nr_objects; i++) {
-			struct object_entry *obj = *list++;
-			uint32_t crc32_val = htonl(obj->crc32);
-			sha1write(f, &crc32_val, 4);
-		}
-
-		/* write the 32-bit offset table */
-		list = sorted_by_sha;
-		for (i = 0; i < nr_objects; i++) {
-			struct object_entry *obj = *list++;
-			uint32_t offset = (obj->offset <= index_off32_limit) ?
-				obj->offset : (0x80000000 | nr_large_offset++);
-			offset = htonl(offset);
-			sha1write(f, &offset, 4);
-		}
-
-		/* write the large offset table */
-		list = sorted_by_sha;
-		while (nr_large_offset) {
-			struct object_entry *obj = *list++;
-			uint64_t offset = obj->offset;
-			if (offset > index_off32_limit) {
-				uint32_t split[2];
-				split[0]	= htonl(offset >> 32);
-				split[1] = htonl(offset & 0xffffffff);
-				sha1write(f, split, 8);
-				nr_large_offset--;
-			}
-		}
-	}
-
-	sha1write(f, sha1, 20);
-	sha1close(f, NULL, 1);
-	free(sorted_by_sha);
-	SHA1_Final(sha1, &ctx);
-	return index_name;
-}
-
 static void final(const char *final_pack_name, const char *curr_pack_name,
 		  const char *final_index_name, const char *curr_index_name,
 		  const char *keep_name, const char *keep_msg,
@@ -775,7 +716,8 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
 				write_or_die(keep_fd, keep_msg, keep_msg_len);
 				write_or_die(keep_fd, "\n", 1);
 			}
-			close(keep_fd);
+			if (close(keep_fd) != 0)
+				die("cannot write keep file");
 			report = "keep";
 		}
 	}
@@ -823,23 +765,39 @@ static void final(const char *final_pack_name, const char *curr_pack_name,
 	}
 }
 
+static int git_index_pack_config(const char *k, const char *v)
+{
+	if (!strcmp(k, "pack.indexversion")) {
+		pack_idx_default_version = git_config_int(k, v);
+		if (pack_idx_default_version > 2)
+			die("bad pack.indexversion=%d", pack_idx_default_version);
+		return 0;
+	}
+	return git_default_config(k, v);
+}
+
 int main(int argc, char **argv)
 {
 	int i, fix_thin_pack = 0;
-	const char *curr_pack, *pack_name = NULL;
-	const char *curr_index, *index_name = NULL;
+	char *curr_pack, *pack_name = NULL;
+	char *curr_index, *index_name = NULL;
 	const char *keep_name = NULL, *keep_msg = NULL;
 	char *index_name_buf = NULL, *keep_name_buf = NULL;
+	struct pack_idx_entry **idx_objects;
 	unsigned char sha1[20];
 
+	git_config(git_index_pack_config);
+
 	for (i = 1; i < argc; i++) {
-		const char *arg = argv[i];
+		char *arg = argv[i];
 
 		if (*arg == '-') {
 			if (!strcmp(arg, "--stdin")) {
 				from_stdin = 1;
 			} else if (!strcmp(arg, "--fix-thin")) {
 				fix_thin_pack = 1;
+			} else if (!strcmp(arg, "--strict")) {
+				strict = 1;
 			} else if (!strcmp(arg, "--keep")) {
 				keep_msg = "";
 			} else if (!prefixcmp(arg, "--keep=")) {
@@ -865,12 +823,12 @@ int main(int argc, char **argv)
 				index_name = argv[++i];
 			} else if (!prefixcmp(arg, "--index-version=")) {
 				char *c;
-				index_default_version = strtoul(arg + 16, &c, 10);
-				if (index_default_version > 2)
+				pack_idx_default_version = strtoul(arg + 16, &c, 10);
+				if (pack_idx_default_version > 2)
 					die("bad %s", arg);
 				if (*c == ',')
-					index_off32_limit = strtoul(c+1, &c, 0);
-				if (*c || index_off32_limit & 0x80000000)
+					pack_idx_off32_limit = strtoul(c+1, &c, 0);
+				if (*c || pack_idx_off32_limit & 0x80000000)
 					die("bad %s", arg);
 			} else
 				usage(index_pack_usage);
@@ -913,12 +871,12 @@ int main(int argc, char **argv)
 	deltas = xmalloc(nr_objects * sizeof(struct delta_entry));
 	parse_pack_objects(sha1);
 	if (nr_deltas == nr_resolved_deltas) {
-		if (verbose)
-			stop_progress(&progress);
+		stop_progress(&progress);
 		/* Flush remaining pack final 20-byte SHA1. */
 		flush();
 	} else {
 		if (fix_thin_pack) {
+			char msg[48];
 			int nr_unresolved = nr_deltas - nr_resolved_deltas;
 			int nr_objects_initial = nr_objects;
 			if (nr_unresolved <= 0)
@@ -927,20 +885,26 @@ int main(int argc, char **argv)
 					   (nr_objects + nr_unresolved + 1)
 					   * sizeof(*objects));
 			fix_unresolved_deltas(nr_unresolved);
-			if (verbose) {
-				stop_progress(&progress);
-				fprintf(stderr, "%d objects were added to complete this thin pack.\n",
-					nr_objects - nr_objects_initial);
-			}
+			sprintf(msg, "completed with %d local objects",
+				nr_objects - nr_objects_initial);
+			stop_progress_msg(&progress, msg);
 			fixup_pack_header_footer(output_fd, sha1,
-				curr_pack, nr_objects);
+						 curr_pack, nr_objects);
 		}
 		if (nr_deltas != nr_resolved_deltas)
 			die("pack has %d unresolved deltas",
 			    nr_deltas - nr_resolved_deltas);
 	}
 	free(deltas);
-	curr_index = write_index_file(index_name, sha1);
+	if (strict)
+		check_objects();
+
+	idx_objects = xmalloc((nr_objects) * sizeof(struct pack_idx_entry *));
+	for (i = 0; i < nr_objects; i++)
+		idx_objects[i] = &objects[i].idx;
+	curr_index = write_idx_file(index_name, idx_objects, nr_objects, sha1);
+	free(idx_objects);
+
 	final(pack_name, curr_pack,
 		index_name, curr_index,
 		keep_name, keep_msg,
@@ -948,6 +912,10 @@ int main(int argc, char **argv)
 	free(objects);
 	free(index_name_buf);
 	free(keep_name_buf);
+	if (pack_name == NULL)
+		free(curr_pack);
+	if (index_name == NULL)
+		free(curr_index);
 
 	return 0;
 }