From: Junio C Hamano <gitster@pobox.com>
Date: Tue, 19 Jul 2011 16:33:03 +0000 (-0700)
Subject: Merge branch 'jc/zlib-wrap'
X-Git-Tag: v1.7.7-rc0~93
X-Git-Url: https://git.tokkee.org/?a=commitdiff_plain;h=eb4f4076aa89d38221ee513a5f28f22124029c90;p=git.git

Merge branch 'jc/zlib-wrap'

* jc/zlib-wrap:
  zlib: allow feeding more than 4GB in one go
  zlib: zlib can only process 4GB at a time
  zlib: wrap deflateBound() too
  zlib: wrap deflate side of the API
  zlib: wrap inflateInit2 used to accept only for gzip format
  zlib: wrap remaining calls to direct inflate/inflateEnd
  zlib wrapper: refactor error message formatter

Conflicts:
	sha1_file.c
---

eb4f4076aa89d38221ee513a5f28f22124029c90
diff --cc cache.h
index bc9e5eb15,1784bab7d..30be90091
--- a/cache.h
+++ b/cache.h
@@@ -752,9 -771,6 +766,9 @@@ extern int hash_sha1_file(const void *b
  extern int write_sha1_file(const void *buf, unsigned long len, const char *type, unsigned char *return_sha1);
  extern int pretend_sha1_file(void *, unsigned long, enum object_type, unsigned char *);
  extern int force_object_loose(const unsigned char *sha1, time_t mtime);
 +extern void *map_sha1_file(const unsigned char *sha1, unsigned long *size);
- extern int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz);
++extern int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz);
 +extern int parse_sha1_header(const char *hdr, unsigned long *sizep);
  
  /* global flag to enable extra checks when accessing packed objects */
  extern int do_check_packed_object_crc;
diff --cc sha1_file.c
index bd6fb2f95,94d431907..a6aac7092
--- a/sha1_file.c
+++ b/sha1_file.c
@@@ -1254,7 -1244,7 +1254,7 @@@ unsigned long unpack_object_header_buff
  	return used;
  }
  
- int unpack_sha1_header(z_stream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
 -static int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
++int unpack_sha1_header(git_zstream *stream, unsigned char *map, unsigned long mapsize, void *buffer, unsigned long bufsiz)
  {
  	unsigned long size, used;
  	static const char valid_loose_object_type[8] = {
@@@ -1271,32 -1261,37 +1271,32 @@@
  	stream->next_out = buffer;
  	stream->avail_out = bufsiz;
  
 -	if (legacy_loose_object(map)) {
 -		git_inflate_init(stream);
 -		return git_inflate(stream, 0);
 -	}
 -
 +	if (experimental_loose_object(map)) {
 +		/*
 +		 * The old experimental format we no longer produce;
 +		 * we can still read it.
 +		 */
 +		used = unpack_object_header_buffer(map, mapsize, &type, &size);
 +		if (!used || !valid_loose_object_type[type])
 +			return -1;
 +		map += used;
 +		mapsize -= used;
  
 -	/*
 -	 * There used to be a second loose object header format which
 -	 * was meant to mimic the in-pack format, allowing for direct
 -	 * copy of the object data.  This format turned up not to be
 -	 * really worth it and we don't write it any longer.  But we
 -	 * can still read it.
 -	 */
 -	used = unpack_object_header_buffer(map, mapsize, &type, &size);
 -	if (!used || !valid_loose_object_type[type])
 -		return -1;
 -	map += used;
 -	mapsize -= used;
 +		/* Set up the stream for the rest.. */
 +		stream->next_in = map;
 +		stream->avail_in = mapsize;
 +		git_inflate_init(stream);
  
 -	/* Set up the stream for the rest.. */
 -	stream->next_in = map;
 -	stream->avail_in = mapsize;
 +		/* And generate the fake traditional header */
 +		stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
 +						 typename(type), size);
 +		return 0;
 +	}
  	git_inflate_init(stream);
 -
 -	/* And generate the fake traditional header */
 -	stream->total_out = 1 + snprintf(buffer, bufsiz, "%s %lu",
 -					 typename(type), size);
 -	return 0;
 +	return git_inflate(stream, 0);
  }
  
- static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
+ static void *unpack_sha1_rest(git_zstream *stream, void *buffer, unsigned long size, const unsigned char *sha1)
  {
  	int bytes = strlen(buffer) + 1;
  	unsigned char *buf = xmallocz(size);
@@@ -1527,13 -1522,13 +1527,13 @@@ static int packed_delta_info(struct pac
  	return type;
  }
  
 -static int unpack_object_header(struct packed_git *p,
 -				struct pack_window **w_curs,
 -				off_t *curpos,
 -				unsigned long *sizep)
 +int unpack_object_header(struct packed_git *p,
 +			 struct pack_window **w_curs,
 +			 off_t *curpos,
 +			 unsigned long *sizep)
  {
  	unsigned char *base;
- 	unsigned int left;
+ 	unsigned long left;
  	unsigned long used;
  	enum object_type type;
  
diff --cc streaming.c
index 91414f459,000000000..25c9a209b
mode 100644,000000..100644
--- a/streaming.c
+++ b/streaming.c
@@@ -1,489 -1,0 +1,489 @@@
 +/*
 + * Copyright (c) 2011, Google Inc.
 + */
 +#include "cache.h"
 +#include "streaming.h"
 +
 +enum input_source {
 +	stream_error = -1,
 +	incore = 0,
 +	loose = 1,
 +	pack_non_delta = 2
 +};
 +
 +typedef int (*open_istream_fn)(struct git_istream *,
 +			       struct object_info *,
 +			       const unsigned char *,
 +			       enum object_type *);
 +typedef int (*close_istream_fn)(struct git_istream *);
 +typedef ssize_t (*read_istream_fn)(struct git_istream *, char *, size_t);
 +
 +struct stream_vtbl {
 +	close_istream_fn close;
 +	read_istream_fn read;
 +};
 +
 +#define open_method_decl(name) \
 +	int open_istream_ ##name \
 +	(struct git_istream *st, struct object_info *oi, \
 +	 const unsigned char *sha1, \
 +	 enum object_type *type)
 +
 +#define close_method_decl(name) \
 +	int close_istream_ ##name \
 +	(struct git_istream *st)
 +
 +#define read_method_decl(name) \
 +	ssize_t read_istream_ ##name \
 +	(struct git_istream *st, char *buf, size_t sz)
 +
 +/* forward declaration */
 +static open_method_decl(incore);
 +static open_method_decl(loose);
 +static open_method_decl(pack_non_delta);
 +static struct git_istream *attach_stream_filter(struct git_istream *st,
 +						struct stream_filter *filter);
 +
 +
 +static open_istream_fn open_istream_tbl[] = {
 +	open_istream_incore,
 +	open_istream_loose,
 +	open_istream_pack_non_delta,
 +};
 +
 +#define FILTER_BUFFER (1024*16)
 +
 +struct filtered_istream {
 +	struct git_istream *upstream;
 +	struct stream_filter *filter;
 +	char ibuf[FILTER_BUFFER];
 +	char obuf[FILTER_BUFFER];
 +	int i_end, i_ptr;
 +	int o_end, o_ptr;
 +	int input_finished;
 +};
 +
 +struct git_istream {
 +	const struct stream_vtbl *vtbl;
 +	unsigned long size; /* inflated size of full object */
- 	z_stream z;
++	git_zstream z;
 +	enum { z_unused, z_used, z_done, z_error } z_state;
 +
 +	union {
 +		struct {
 +			char *buf; /* from read_object() */
 +			unsigned long read_ptr;
 +		} incore;
 +
 +		struct {
 +			void *mapped;
 +			unsigned long mapsize;
 +			char hdr[32];
 +			int hdr_avail;
 +			int hdr_used;
 +		} loose;
 +
 +		struct {
 +			struct packed_git *pack;
 +			off_t pos;
 +		} in_pack;
 +
 +		struct filtered_istream filtered;
 +	} u;
 +};
 +
 +int close_istream(struct git_istream *st)
 +{
 +	return st->vtbl->close(st);
 +}
 +
 +ssize_t read_istream(struct git_istream *st, char *buf, size_t sz)
 +{
 +	return st->vtbl->read(st, buf, sz);
 +}
 +
 +static enum input_source istream_source(const unsigned char *sha1,
 +					enum object_type *type,
 +					struct object_info *oi)
 +{
 +	unsigned long size;
 +	int status;
 +
 +	oi->sizep = &size;
 +	status = sha1_object_info_extended(sha1, oi);
 +	if (status < 0)
 +		return stream_error;
 +	*type = status;
 +
 +	switch (oi->whence) {
 +	case OI_LOOSE:
 +		return loose;
 +	case OI_PACKED:
 +		if (!oi->u.packed.is_delta && big_file_threshold <= size)
 +			return pack_non_delta;
 +		/* fallthru */
 +	default:
 +		return incore;
 +	}
 +}
 +
 +struct git_istream *open_istream(const unsigned char *sha1,
 +				 enum object_type *type,
 +				 unsigned long *size,
 +				 struct stream_filter *filter)
 +{
 +	struct git_istream *st;
 +	struct object_info oi;
 +	const unsigned char *real = lookup_replace_object(sha1);
 +	enum input_source src = istream_source(real, type, &oi);
 +
 +	if (src < 0)
 +		return NULL;
 +
 +	st = xmalloc(sizeof(*st));
 +	if (open_istream_tbl[src](st, &oi, real, type)) {
 +		if (open_istream_incore(st, &oi, real, type)) {
 +			free(st);
 +			return NULL;
 +		}
 +	}
 +	if (st && filter) {
 +		/* Add "&& !is_null_stream_filter(filter)" for performance */
 +		struct git_istream *nst = attach_stream_filter(st, filter);
 +		if (!nst)
 +			close_istream(st);
 +		st = nst;
 +	}
 +
 +	*size = st->size;
 +	return st;
 +}
 +
 +
 +/*****************************************************************
 + *
 + * Common helpers
 + *
 + *****************************************************************/
 +
 +static void close_deflated_stream(struct git_istream *st)
 +{
 +	if (st->z_state == z_used)
 +		git_inflate_end(&st->z);
 +}
 +
 +
 +/*****************************************************************
 + *
 + * Filtered stream
 + *
 + *****************************************************************/
 +
 +static close_method_decl(filtered)
 +{
 +	free_stream_filter(st->u.filtered.filter);
 +	return close_istream(st->u.filtered.upstream);
 +}
 +
 +static read_method_decl(filtered)
 +{
 +	struct filtered_istream *fs = &(st->u.filtered);
 +	size_t filled = 0;
 +
 +	while (sz) {
 +		/* do we already have filtered output? */
 +		if (fs->o_ptr < fs->o_end) {
 +			size_t to_move = fs->o_end - fs->o_ptr;
 +			if (sz < to_move)
 +				to_move = sz;
 +			memcpy(buf + filled, fs->obuf + fs->o_ptr, to_move);
 +			fs->o_ptr += to_move;
 +			sz -= to_move;
 +			filled += to_move;
 +			continue;
 +		}
 +		fs->o_end = fs->o_ptr = 0;
 +
 +		/* do we have anything to feed the filter with? */
 +		if (fs->i_ptr < fs->i_end) {
 +			size_t to_feed = fs->i_end - fs->i_ptr;
 +			size_t to_receive = FILTER_BUFFER;
 +			if (stream_filter(fs->filter,
 +					  fs->ibuf + fs->i_ptr, &to_feed,
 +					  fs->obuf, &to_receive))
 +				return -1;
 +			fs->i_ptr = fs->i_end - to_feed;
 +			fs->o_end = FILTER_BUFFER - to_receive;
 +			continue;
 +		}
 +
 +		/* tell the filter to drain upon no more input */
 +		if (fs->input_finished) {
 +			size_t to_receive = FILTER_BUFFER;
 +			if (stream_filter(fs->filter,
 +					  NULL, NULL,
 +					  fs->obuf, &to_receive))
 +				return -1;
 +			fs->o_end = FILTER_BUFFER - to_receive;
 +			if (!fs->o_end)
 +				break;
 +			continue;
 +		}
 +		fs->i_end = fs->i_ptr = 0;
 +
 +		/* refill the input from the upstream */
 +		if (!fs->input_finished) {
 +			fs->i_end = read_istream(fs->upstream, fs->ibuf, FILTER_BUFFER);
 +			if (fs->i_end < 0)
 +				break;
 +			if (fs->i_end)
 +				continue;
 +		}
 +		fs->input_finished = 1;
 +	}
 +	return filled;
 +}
 +
 +static struct stream_vtbl filtered_vtbl = {
 +	close_istream_filtered,
 +	read_istream_filtered,
 +};
 +
 +static struct git_istream *attach_stream_filter(struct git_istream *st,
 +						struct stream_filter *filter)
 +{
 +	struct git_istream *ifs = xmalloc(sizeof(*ifs));
 +	struct filtered_istream *fs = &(ifs->u.filtered);
 +
 +	ifs->vtbl = &filtered_vtbl;
 +	fs->upstream = st;
 +	fs->filter = filter;
 +	fs->i_end = fs->i_ptr = 0;
 +	fs->o_end = fs->o_ptr = 0;
 +	fs->input_finished = 0;
 +	ifs->size = -1; /* unknown */
 +	return ifs;
 +}
 +
 +/*****************************************************************
 + *
 + * Loose object stream
 + *
 + *****************************************************************/
 +
 +static read_method_decl(loose)
 +{
 +	size_t total_read = 0;
 +
 +	switch (st->z_state) {
 +	case z_done:
 +		return 0;
 +	case z_error:
 +		return -1;
 +	default:
 +		break;
 +	}
 +
 +	if (st->u.loose.hdr_used < st->u.loose.hdr_avail) {
 +		size_t to_copy = st->u.loose.hdr_avail - st->u.loose.hdr_used;
 +		if (sz < to_copy)
 +			to_copy = sz;
 +		memcpy(buf, st->u.loose.hdr + st->u.loose.hdr_used, to_copy);
 +		st->u.loose.hdr_used += to_copy;
 +		total_read += to_copy;
 +	}
 +
 +	while (total_read < sz) {
 +		int status;
 +
 +		st->z.next_out = (unsigned char *)buf + total_read;
 +		st->z.avail_out = sz - total_read;
 +		status = git_inflate(&st->z, Z_FINISH);
 +
 +		total_read = st->z.next_out - (unsigned char *)buf;
 +
 +		if (status == Z_STREAM_END) {
 +			git_inflate_end(&st->z);
 +			st->z_state = z_done;
 +			break;
 +		}
 +		if (status != Z_OK && status != Z_BUF_ERROR) {
 +			git_inflate_end(&st->z);
 +			st->z_state = z_error;
 +			return -1;
 +		}
 +	}
 +	return total_read;
 +}
 +
 +static close_method_decl(loose)
 +{
 +	close_deflated_stream(st);
 +	munmap(st->u.loose.mapped, st->u.loose.mapsize);
 +	return 0;
 +}
 +
 +static struct stream_vtbl loose_vtbl = {
 +	close_istream_loose,
 +	read_istream_loose,
 +};
 +
 +static open_method_decl(loose)
 +{
 +	st->u.loose.mapped = map_sha1_file(sha1, &st->u.loose.mapsize);
 +	if (!st->u.loose.mapped)
 +		return -1;
 +	if (unpack_sha1_header(&st->z,
 +			       st->u.loose.mapped,
 +			       st->u.loose.mapsize,
 +			       st->u.loose.hdr,
 +			       sizeof(st->u.loose.hdr)) < 0) {
 +		git_inflate_end(&st->z);
 +		munmap(st->u.loose.mapped, st->u.loose.mapsize);
 +		return -1;
 +	}
 +
 +	parse_sha1_header(st->u.loose.hdr, &st->size);
 +	st->u.loose.hdr_used = strlen(st->u.loose.hdr) + 1;
 +	st->u.loose.hdr_avail = st->z.total_out;
 +	st->z_state = z_used;
 +
 +	st->vtbl = &loose_vtbl;
 +	return 0;
 +}
 +
 +
 +/*****************************************************************
 + *
 + * Non-delta packed object stream
 + *
 + *****************************************************************/
 +
 +static read_method_decl(pack_non_delta)
 +{
 +	size_t total_read = 0;
 +
 +	switch (st->z_state) {
 +	case z_unused:
 +		memset(&st->z, 0, sizeof(st->z));
 +		git_inflate_init(&st->z);
 +		st->z_state = z_used;
 +		break;
 +	case z_done:
 +		return 0;
 +	case z_error:
 +		return -1;
 +	case z_used:
 +		break;
 +	}
 +
 +	while (total_read < sz) {
 +		int status;
 +		struct pack_window *window = NULL;
 +		unsigned char *mapped;
 +
 +		mapped = use_pack(st->u.in_pack.pack, &window,
 +				  st->u.in_pack.pos, &st->z.avail_in);
 +
 +		st->z.next_out = (unsigned char *)buf + total_read;
 +		st->z.avail_out = sz - total_read;
 +		st->z.next_in = mapped;
 +		status = git_inflate(&st->z, Z_FINISH);
 +
 +		st->u.in_pack.pos += st->z.next_in - mapped;
 +		total_read = st->z.next_out - (unsigned char *)buf;
 +		unuse_pack(&window);
 +
 +		if (status == Z_STREAM_END) {
 +			git_inflate_end(&st->z);
 +			st->z_state = z_done;
 +			break;
 +		}
 +		if (status != Z_OK && status != Z_BUF_ERROR) {
 +			git_inflate_end(&st->z);
 +			st->z_state = z_error;
 +			return -1;
 +		}
 +	}
 +	return total_read;
 +}
 +
 +static close_method_decl(pack_non_delta)
 +{
 +	close_deflated_stream(st);
 +	return 0;
 +}
 +
 +static struct stream_vtbl pack_non_delta_vtbl = {
 +	close_istream_pack_non_delta,
 +	read_istream_pack_non_delta,
 +};
 +
 +static open_method_decl(pack_non_delta)
 +{
 +	struct pack_window *window;
 +	enum object_type in_pack_type;
 +
 +	st->u.in_pack.pack = oi->u.packed.pack;
 +	st->u.in_pack.pos = oi->u.packed.offset;
 +	window = NULL;
 +
 +	in_pack_type = unpack_object_header(st->u.in_pack.pack,
 +					    &window,
 +					    &st->u.in_pack.pos,
 +					    &st->size);
 +	unuse_pack(&window);
 +	switch (in_pack_type) {
 +	default:
 +		return -1; /* we do not do deltas for now */
 +	case OBJ_COMMIT:
 +	case OBJ_TREE:
 +	case OBJ_BLOB:
 +	case OBJ_TAG:
 +		break;
 +	}
 +	st->z_state = z_unused;
 +	st->vtbl = &pack_non_delta_vtbl;
 +	return 0;
 +}
 +
 +
 +/*****************************************************************
 + *
 + * In-core stream
 + *
 + *****************************************************************/
 +
 +static close_method_decl(incore)
 +{
 +	free(st->u.incore.buf);
 +	return 0;
 +}
 +
 +static read_method_decl(incore)
 +{
 +	size_t read_size = sz;
 +	size_t remainder = st->size - st->u.incore.read_ptr;
 +
 +	if (remainder <= read_size)
 +		read_size = remainder;
 +	if (read_size) {
 +		memcpy(buf, st->u.incore.buf + st->u.incore.read_ptr, read_size);
 +		st->u.incore.read_ptr += read_size;
 +	}
 +	return read_size;
 +}
 +
 +static struct stream_vtbl incore_vtbl = {
 +	close_istream_incore,
 +	read_istream_incore,
 +};
 +
 +static open_method_decl(incore)
 +{
 +	st->u.incore.buf = read_sha1_file_extended(sha1, type, &st->size, 0);
 +	st->u.incore.read_ptr = 0;
 +	st->vtbl = &incore_vtbl;
 +
 +	return st->u.incore.buf ? 0 : -1;
 +}