streaming.c

   1 /*
   2  * Copyright (c) 2011, Google Inc.
   3  */
   4 #include "cache.h"
   5 #include "streaming.h"
   6
   7 enum input_source {
   8         stream_error = -1,
   9         incore = 0,
  10         loose = 1,
  11         pack_non_delta = 2
  12 };
  13
  14 typedef int (*open_istream_fn)(struct git_istream *,
  15                                struct object_info *,
  16                                const unsigned char *,
  17                                enum object_type *);
  18 typedef int (*close_istream_fn)(struct git_istream *);
  19 typedef ssize_t (*read_istream_fn)(struct git_istream *, char *, size_t);
  20
  21 struct stream_vtbl {
  22         close_istream_fn close;
  23         read_istream_fn read;
  24 };
  25
  26 #define open_method_decl(name) \
  27         int open_istream_ ##name \
  28         (struct git_istream *st, struct object_info *oi, \
  29          const unsigned char *sha1, \
  30          enum object_type *type)
  31
  32 #define close_method_decl(name) \
  33         int close_istream_ ##name \
  34         (struct git_istream *st)
  35
  36 #define read_method_decl(name) \
  37         ssize_t read_istream_ ##name \
  38         (struct git_istream *st, char *buf, size_t sz)
  39
  40 /* forward declaration */
  41 static open_method_decl(incore);
  42 static open_method_decl(loose);
  43 static open_method_decl(pack_non_delta);
  44
  45 static open_istream_fn open_istream_tbl[] = {
  46         open_istream_incore,
  47         open_istream_loose,
  48         open_istream_pack_non_delta,
  49 };
  50
  51 struct git_istream {
  52         const struct stream_vtbl *vtbl;
  53         unsigned long size; /* inflated size of full object */
  54         z_stream z;
  55         enum { z_unused, z_used, z_done, z_error } z_state;
  56
  57         union {
  58                 struct {
  59                         char *buf; /* from read_object() */
  60                         unsigned long read_ptr;
  61                 } incore;
  62
  63                 struct {
  64                         void *mapped;
  65                         unsigned long mapsize;
  66                         char hdr[32];
  67                         int hdr_avail;
  68                         int hdr_used;
  69                 } loose;
  70
  71                 struct {
  72                         struct packed_git *pack;
  73                         off_t pos;
  74                 } in_pack;
  75         } u;
  76 };
  77
  78 int close_istream(struct git_istream *st)
  79 {
  80         return st->vtbl->close(st);
  81 }
  82
  83 ssize_t read_istream(struct git_istream *st, char *buf, size_t sz)
  84 {
  85         return st->vtbl->read(st, buf, sz);
  86 }
  87
  88 static enum input_source istream_source(const unsigned char *sha1,
  89                                         enum object_type *type,
  90                                         struct object_info *oi)
  91 {
  92         unsigned long size;
  93         int status;
  94
  95         oi->sizep = &size;
  96         status = sha1_object_info_extended(sha1, oi);
  97         if (status < 0)
  98                 return stream_error;
  99         *type = status;
 100
 101         switch (oi->whence) {
 102         case OI_LOOSE:
 103                 return loose;
 104         case OI_PACKED:
 105                 if (!oi->u.packed.is_delta && big_file_threshold <= size)
 106                         return pack_non_delta;
 107                 /* fallthru */
 108         default:
 109                 return incore;
 110         }
 111 }
 112
 113 struct git_istream *open_istream(const unsigned char *sha1,
 114                                  enum object_type *type,
 115                                  unsigned long *size)
 116 {
 117         struct git_istream *st;
 118         struct object_info oi;
 119         const unsigned char *real = lookup_replace_object(sha1);
 120         enum input_source src = istream_source(real, type, &oi);
 121
 122         if (src < 0)
 123                 return NULL;
 124
 125         st = xmalloc(sizeof(*st));
 126         if (open_istream_tbl[src](st, &oi, real, type)) {
 127                 if (open_istream_incore(st, &oi, real, type)) {
 128                         free(st);
 129                         return NULL;
 130                 }
 131         }
 132         *size = st->size;
 133         return st;
 134 }
 135
 136
 137 /*****************************************************************
 138  *
 139  * Common helpers
 140  *
 141  *****************************************************************/
 142
 143 static void close_deflated_stream(struct git_istream *st)
 144 {
 145         if (st->z_state == z_used)
 146                 git_inflate_end(&st->z);
 147 }
 148
 149
 150 /*****************************************************************
 151  *
 152  * Loose object stream
 153  *
 154  *****************************************************************/
 155
 156 static read_method_decl(loose)
 157 {
 158         size_t total_read = 0;
 159
 160         switch (st->z_state) {
 161         case z_done:
 162                 return 0;
 163         case z_error:
 164                 return -1;
 165         default:
 166                 break;
 167         }
 168
 169         if (st->u.loose.hdr_used < st->u.loose.hdr_avail) {
 170                 size_t to_copy = st->u.loose.hdr_avail - st->u.loose.hdr_used;
 171                 if (sz < to_copy)
 172                         to_copy = sz;
 173                 memcpy(buf, st->u.loose.hdr + st->u.loose.hdr_used, to_copy);
 174                 st->u.loose.hdr_used += to_copy;
 175                 total_read += to_copy;
 176         }
 177
 178         while (total_read < sz) {
 179                 int status;
 180
 181                 st->z.next_out = (unsigned char *)buf + total_read;
 182                 st->z.avail_out = sz - total_read;
 183                 status = git_inflate(&st->z, Z_FINISH);
 184
 185                 total_read = st->z.next_out - (unsigned char *)buf;
 186
 187                 if (status == Z_STREAM_END) {
 188                         git_inflate_end(&st->z);
 189                         st->z_state = z_done;
 190                         break;
 191                 }
 192                 if (status != Z_OK && status != Z_BUF_ERROR) {
 193                         git_inflate_end(&st->z);
 194                         st->z_state = z_error;
 195                         return -1;
 196                 }
 197         }
 198         return total_read;
 199 }
 200
 201 static close_method_decl(loose)
 202 {
 203         close_deflated_stream(st);
 204         munmap(st->u.loose.mapped, st->u.loose.mapsize);
 205         return 0;
 206 }
 207
 208 static struct stream_vtbl loose_vtbl = {
 209         close_istream_loose,
 210         read_istream_loose,
 211 };
 212
 213 static open_method_decl(loose)
 214 {
 215         st->u.loose.mapped = map_sha1_file(sha1, &st->u.loose.mapsize);
 216         if (!st->u.loose.mapped)
 217                 return -1;
 218         if (unpack_sha1_header(&st->z,
 219                                st->u.loose.mapped,
 220                                st->u.loose.mapsize,
 221                                st->u.loose.hdr,
 222                                sizeof(st->u.loose.hdr)) < 0) {
 223                 git_inflate_end(&st->z);
 224                 munmap(st->u.loose.mapped, st->u.loose.mapsize);
 225                 return -1;
 226         }
 227
 228         parse_sha1_header(st->u.loose.hdr, &st->size);
 229         st->u.loose.hdr_used = strlen(st->u.loose.hdr) + 1;
 230         st->u.loose.hdr_avail = st->z.total_out;
 231         st->z_state = z_used;
 232
 233         st->vtbl = &loose_vtbl;
 234         return 0;
 235 }
 236
 237
 238 /*****************************************************************
 239  *
 240  * Non-delta packed object stream
 241  *
 242  *****************************************************************/
 243
 244 static read_method_decl(pack_non_delta)
 245 {
 246         size_t total_read = 0;
 247
 248         switch (st->z_state) {
 249         case z_unused:
 250                 memset(&st->z, 0, sizeof(st->z));
 251                 git_inflate_init(&st->z);
 252                 st->z_state = z_used;
 253                 break;
 254         case z_done:
 255                 return 0;
 256         case z_error:
 257                 return -1;
 258         case z_used:
 259                 break;
 260         }
 261
 262         while (total_read < sz) {
 263                 int status;
 264                 struct pack_window *window = NULL;
 265                 unsigned char *mapped;
 266
 267                 mapped = use_pack(st->u.in_pack.pack, &window,
 268                                   st->u.in_pack.pos, &st->z.avail_in);
 269
 270                 st->z.next_out = (unsigned char *)buf + total_read;
 271                 st->z.avail_out = sz - total_read;
 272                 st->z.next_in = mapped;
 273                 status = git_inflate(&st->z, Z_FINISH);
 274
 275                 st->u.in_pack.pos += st->z.next_in - mapped;
 276                 total_read = st->z.next_out - (unsigned char *)buf;
 277                 unuse_pack(&window);
 278
 279                 if (status == Z_STREAM_END) {
 280                         git_inflate_end(&st->z);
 281                         st->z_state = z_done;
 282                         break;
 283                 }
 284                 if (status != Z_OK && status != Z_BUF_ERROR) {
 285                         git_inflate_end(&st->z);
 286                         st->z_state = z_error;
 287                         return -1;
 288                 }
 289         }
 290         return total_read;
 291 }
 292
 293 static close_method_decl(pack_non_delta)
 294 {
 295         close_deflated_stream(st);
 296         return 0;
 297 }
 298
 299 static struct stream_vtbl pack_non_delta_vtbl = {
 300         close_istream_pack_non_delta,
 301         read_istream_pack_non_delta,
 302 };
 303
 304 static open_method_decl(pack_non_delta)
 305 {
 306         struct pack_window *window;
 307         enum object_type in_pack_type;
 308
 309         st->u.in_pack.pack = oi->u.packed.pack;
 310         st->u.in_pack.pos = oi->u.packed.offset;
 311         window = NULL;
 312
 313         in_pack_type = unpack_object_header(st->u.in_pack.pack,
 314                                             &window,
 315                                             &st->u.in_pack.pos,
 316                                             &st->size);
 317         unuse_pack(&window);
 318         switch (in_pack_type) {
 319         default:
 320                 return -1; /* we do not do deltas for now */
 321         case OBJ_COMMIT:
 322         case OBJ_TREE:
 323         case OBJ_BLOB:
 324         case OBJ_TAG:
 325                 break;
 326         }
 327         st->z_state = z_unused;
 328         st->vtbl = &pack_non_delta_vtbl;
 329         return 0;
 330 }
 331
 332
 333 /*****************************************************************
 334  *
 335  * In-core stream
 336  *
 337  *****************************************************************/
 338
 339 static close_method_decl(incore)
 340 {
 341         free(st->u.incore.buf);
 342         return 0;
 343 }
 344
 345 static read_method_decl(incore)
 346 {
 347         size_t read_size = sz;
 348         size_t remainder = st->size - st->u.incore.read_ptr;
 349
 350         if (remainder <= read_size)
 351                 read_size = remainder;
 352         if (read_size) {
 353                 memcpy(buf, st->u.incore.buf + st->u.incore.read_ptr, read_size);
 354                 st->u.incore.read_ptr += read_size;
 355         }
 356         return read_size;
 357 }
 358
 359 static struct stream_vtbl incore_vtbl = {
 360         close_istream_incore,
 361         read_istream_incore,
 362 };
 363
 364 static open_method_decl(incore)
 365 {
 366         st->u.incore.buf = read_sha1_file_extended(sha1, type, &st->size, 0);
 367         st->u.incore.read_ptr = 0;
 368         st->vtbl = &incore_vtbl;
 369
 370         return st->u.incore.buf ? 0 : -1;
 371 }