builtin-unpack-objects.c

   1 #include "builtin.h"
   2 #include "cache.h"
   3 #include "object.h"
   4 #include "delta.h"
   5 #include "pack.h"
   6 #include "blob.h"
   7 #include "commit.h"
   8 #include "tag.h"
   9 #include "tree.h"
  10 #include "tree-walk.h"
  11 #include "progress.h"
  12 #include "decorate.h"
  13 #include "fsck.h"
  14
  15 static int dry_run, quiet, recover, has_errors, strict;
  16 static const char unpack_usage[] = "git-unpack-objects [-n] [-q] [-r] [--strict] < pack-file";
  17
  18 /* We always read in 4kB chunks. */
  19 static unsigned char buffer[4096];
  20 static unsigned int offset, len;
  21 static off_t consumed_bytes;
  22 static SHA_CTX ctx;
  23
  24 struct obj_buffer {
  25         char *buffer;
  26         unsigned long size;
  27 };
  28
  29 static struct decoration obj_decorate;
  30
  31 static struct obj_buffer *lookup_object_buffer(struct object *base)
  32 {
  33         return lookup_decoration(&obj_decorate, base);
  34 }
  35
  36 static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
  37 {
  38         struct obj_buffer *obj;
  39         obj = xcalloc(1, sizeof(struct obj_buffer));
  40         obj->buffer = buffer;
  41         obj->size = size;
  42         if (add_decoration(&obj_decorate, object, obj))
  43                 die("object %s tried to add buffer twice!", sha1_to_hex(object->sha1));
  44 }
  45
  46 /*
  47  * Make sure at least "min" bytes are available in the buffer, and
  48  * return the pointer to the buffer.
  49  */
  50 static void *fill(int min)
  51 {
  52         if (min <= len)
  53                 return buffer + offset;
  54         if (min > sizeof(buffer))
  55                 die("cannot fill %d bytes", min);
  56         if (offset) {
  57                 SHA1_Update(&ctx, buffer, offset);
  58                 memmove(buffer, buffer + offset, len);
  59                 offset = 0;
  60         }
  61         do {
  62                 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
  63                 if (ret <= 0) {
  64                         if (!ret)
  65                                 die("early EOF");
  66                         die("read error on input: %s", strerror(errno));
  67                 }
  68                 len += ret;
  69         } while (len < min);
  70         return buffer;
  71 }
  72
  73 static void use(int bytes)
  74 {
  75         if (bytes > len)
  76                 die("used more bytes than were available");
  77         len -= bytes;
  78         offset += bytes;
  79
  80         /* make sure off_t is sufficiently large not to wrap */
  81         if (consumed_bytes > consumed_bytes + bytes)
  82                 die("pack too large for current definition of off_t");
  83         consumed_bytes += bytes;
  84 }
  85
  86 static void *get_data(unsigned long size)
  87 {
  88         z_stream stream;
  89         void *buf = xmalloc(size);
  90
  91         memset(&stream, 0, sizeof(stream));
  92
  93         stream.next_out = buf;
  94         stream.avail_out = size;
  95         stream.next_in = fill(1);
  96         stream.avail_in = len;
  97         inflateInit(&stream);
  98
  99         for (;;) {
 100                 int ret = inflate(&stream, 0);
 101                 use(len - stream.avail_in);
 102                 if (stream.total_out == size && ret == Z_STREAM_END)
 103                         break;
 104                 if (ret != Z_OK) {
 105                         error("inflate returned %d\n", ret);
 106                         free(buf);
 107                         buf = NULL;
 108                         if (!recover)
 109                                 exit(1);
 110                         has_errors = 1;
 111                         break;
 112                 }
 113                 stream.next_in = fill(1);
 114                 stream.avail_in = len;
 115         }
 116         inflateEnd(&stream);
 117         return buf;
 118 }
 119
 120 struct delta_info {
 121         unsigned char base_sha1[20];
 122         unsigned nr;
 123         off_t base_offset;
 124         unsigned long size;
 125         void *delta;
 126         struct delta_info *next;
 127 };
 128
 129 static struct delta_info *delta_list;
 130
 131 static void add_delta_to_list(unsigned nr, unsigned const char *base_sha1,
 132                               off_t base_offset,
 133                               void *delta, unsigned long size)
 134 {
 135         struct delta_info *info = xmalloc(sizeof(*info));
 136
 137         hashcpy(info->base_sha1, base_sha1);
 138         info->base_offset = base_offset;
 139         info->size = size;
 140         info->delta = delta;
 141         info->nr = nr;
 142         info->next = delta_list;
 143         delta_list = info;
 144 }
 145
 146 struct obj_info {
 147         off_t offset;
 148         unsigned char sha1[20];
 149         struct object *obj;
 150 };
 151
 152 #define FLAG_OPEN (1u<<20)
 153 #define FLAG_WRITTEN (1u<<21)
 154
 155 static struct obj_info *obj_list;
 156 unsigned nr_objects;
 157
 158 static void write_cached_object(struct object *obj)
 159 {
 160         unsigned char sha1[20];
 161         struct obj_buffer *obj_buf = lookup_object_buffer(obj);
 162         if (write_sha1_file(obj_buf->buffer, obj_buf->size, typename(obj->type), sha1) < 0)
 163                 die("failed to write object %s", sha1_to_hex(obj->sha1));
 164         obj->flags |= FLAG_WRITTEN;
 165 }
 166
 167 static int check_object(struct object *obj, int type, void *data)
 168 {
 169         if (!obj)
 170                 return 0;
 171
 172         if (obj->flags & FLAG_WRITTEN)
 173                 return 1;
 174
 175         if (type != OBJ_ANY && obj->type != type)
 176                 die("object type mismatch");
 177
 178         if (!(obj->flags & FLAG_OPEN)) {
 179                 unsigned long size;
 180                 int type = sha1_object_info(obj->sha1, &size);
 181                 if (type != obj->type || type <= 0)
 182                         die("object of unexpected type");
 183                 obj->flags |= FLAG_WRITTEN;
 184                 return 1;
 185         }
 186
 187         if (fsck_object(obj, 1, fsck_error_function))
 188                 die("Error in object");
 189         if (!fsck_walk(obj, check_object, 0))
 190                 die("Error on reachable objects of %s", sha1_to_hex(obj->sha1));
 191         write_cached_object(obj);
 192         return 1;
 193 }
 194
 195 static void write_rest(void)
 196 {
 197         unsigned i;
 198         for (i = 0; i < nr_objects; i++)
 199                 check_object(obj_list[i].obj, OBJ_ANY, 0);
 200 }
 201
 202 static void added_object(unsigned nr, enum object_type type,
 203                          void *data, unsigned long size);
 204
 205 static void write_object(unsigned nr, enum object_type type,
 206                          void *buf, unsigned long size)
 207 {
 208         added_object(nr, type, buf, size);
 209         if (!strict) {
 210                 if (write_sha1_file(buf, size, typename(type), obj_list[nr].sha1) < 0)
 211                         die("failed to write object");
 212                 free(buf);
 213                 obj_list[nr].obj = 0;
 214         } else if (type == OBJ_BLOB) {
 215                 struct blob *blob;
 216                 if (write_sha1_file(buf, size, typename(type), obj_list[nr].sha1) < 0)
 217                         die("failed to write object");
 218                 free(buf);
 219
 220                 blob = lookup_blob(obj_list[nr].sha1);
 221                 if (blob)
 222                         blob->object.flags |= FLAG_WRITTEN;
 223                 else
 224                         die("invalid blob object");
 225                 obj_list[nr].obj = 0;
 226         } else {
 227                 struct object *obj;
 228                 int eaten;
 229                 hash_sha1_file(buf, size, typename(type), obj_list[nr].sha1);
 230                 obj = parse_object_buffer(obj_list[nr].sha1, type, size, buf, &eaten);
 231                 if (!obj)
 232                         die("invalid %s", typename(type));
 233                 /* buf is stored via add_object_buffer and in obj, if its a tree or commit */
 234                 add_object_buffer(obj, buf, size);
 235                 obj->flags |= FLAG_OPEN;
 236                 obj_list[nr].obj = obj;
 237         }
 238 }
 239
 240 static void resolve_delta(unsigned nr, enum object_type type,
 241                           void *base, unsigned long base_size,
 242                           void *delta, unsigned long delta_size)
 243 {
 244         void *result;
 245         unsigned long result_size;
 246
 247         result = patch_delta(base, base_size,
 248                              delta, delta_size,
 249                              &result_size);
 250         if (!result)
 251                 die("failed to apply delta");
 252         free(delta);
 253         write_object(nr, type, result, result_size);
 254 }
 255
 256 static void added_object(unsigned nr, enum object_type type,
 257                          void *data, unsigned long size)
 258 {
 259         struct delta_info **p = &delta_list;
 260         struct delta_info *info;
 261
 262         while ((info = *p) != NULL) {
 263                 if (!hashcmp(info->base_sha1, obj_list[nr].sha1) ||
 264                     info->base_offset == obj_list[nr].offset) {
 265                         *p = info->next;
 266                         p = &delta_list;
 267                         resolve_delta(info->nr, type, data, size,
 268                                       info->delta, info->size);
 269                         free(info);
 270                         continue;
 271                 }
 272                 p = &info->next;
 273         }
 274 }
 275
 276 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
 277                                    unsigned nr)
 278 {
 279         void *buf = get_data(size);
 280
 281         if (!dry_run && buf)
 282                 write_object(nr, type, buf, size);
 283         else
 284                 free(buf);
 285 }
 286
 287 static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
 288                                unsigned nr)
 289 {
 290         void *delta_data, *base;
 291         unsigned long base_size;
 292         unsigned char base_sha1[20];
 293         struct object *obj;
 294
 295         if (type == OBJ_REF_DELTA) {
 296                 hashcpy(base_sha1, fill(20));
 297                 use(20);
 298                 delta_data = get_data(delta_size);
 299                 if (dry_run || !delta_data) {
 300                         free(delta_data);
 301                         return;
 302                 }
 303                 if (!has_sha1_file(base_sha1)) {
 304                         hashcpy(obj_list[nr].sha1, null_sha1);
 305                         add_delta_to_list(nr, base_sha1, 0, delta_data, delta_size);
 306                         return;
 307                 }
 308         } else {
 309                 unsigned base_found = 0;
 310                 unsigned char *pack, c;
 311                 off_t base_offset;
 312                 unsigned lo, mid, hi;
 313
 314                 pack = fill(1);
 315                 c = *pack;
 316                 use(1);
 317                 base_offset = c & 127;
 318                 while (c & 128) {
 319                         base_offset += 1;
 320                         if (!base_offset || MSB(base_offset, 7))
 321                                 die("offset value overflow for delta base object");
 322                         pack = fill(1);
 323                         c = *pack;
 324                         use(1);
 325                         base_offset = (base_offset << 7) + (c & 127);
 326                 }
 327                 base_offset = obj_list[nr].offset - base_offset;
 328
 329                 delta_data = get_data(delta_size);
 330                 if (dry_run || !delta_data) {
 331                         free(delta_data);
 332                         return;
 333                 }
 334                 lo = 0;
 335                 hi = nr;
 336                 while (lo < hi) {
 337                         mid = (lo + hi)/2;
 338                         if (base_offset < obj_list[mid].offset) {
 339                                 hi = mid;
 340                         } else if (base_offset > obj_list[mid].offset) {
 341                                 lo = mid + 1;
 342                         } else {
 343                                 hashcpy(base_sha1, obj_list[mid].sha1);
 344                                 base_found = !is_null_sha1(base_sha1);
 345                                 break;
 346                         }
 347                 }
 348                 if (!base_found) {
 349                         /* The delta base object is itself a delta that
 350                            has not been resolved yet. */
 351                         hashcpy(obj_list[nr].sha1, null_sha1);
 352                         add_delta_to_list(nr, null_sha1, base_offset, delta_data, delta_size);
 353                         return;
 354                 }
 355         }
 356
 357         obj = lookup_object(base_sha1);
 358         if (obj) {
 359                 struct obj_buffer *obj_buf = lookup_object_buffer(obj);
 360                 if (obj_buf) {
 361                         resolve_delta(nr, obj->type, obj_buf->buffer, obj_buf->size, delta_data, delta_size);
 362                         return;
 363                 }
 364         }
 365
 366         base = read_sha1_file(base_sha1, &type, &base_size);
 367         if (!base) {
 368                 error("failed to read delta-pack base object %s",
 369                       sha1_to_hex(base_sha1));
 370                 if (!recover)
 371                         exit(1);
 372                 has_errors = 1;
 373                 return;
 374         }
 375         resolve_delta(nr, type, base, base_size, delta_data, delta_size);
 376         free(base);
 377 }
 378
 379 static void unpack_one(unsigned nr)
 380 {
 381         unsigned shift;
 382         unsigned char *pack, c;
 383         unsigned long size;
 384         enum object_type type;
 385
 386         obj_list[nr].offset = consumed_bytes;
 387
 388         pack = fill(1);
 389         c = *pack;
 390         use(1);
 391         type = (c >> 4) & 7;
 392         size = (c & 15);
 393         shift = 4;
 394         while (c & 0x80) {
 395                 pack = fill(1);
 396                 c = *pack;
 397                 use(1);
 398                 size += (c & 0x7f) << shift;
 399                 shift += 7;
 400         }
 401
 402         switch (type) {
 403         case OBJ_COMMIT:
 404         case OBJ_TREE:
 405         case OBJ_BLOB:
 406         case OBJ_TAG:
 407                 unpack_non_delta_entry(type, size, nr);
 408                 return;
 409         case OBJ_REF_DELTA:
 410         case OBJ_OFS_DELTA:
 411                 unpack_delta_entry(type, size, nr);
 412                 return;
 413         default:
 414                 error("bad object type %d", type);
 415                 has_errors = 1;
 416                 if (recover)
 417                         return;
 418                 exit(1);
 419         }
 420 }
 421
 422 static void unpack_all(void)
 423 {
 424         int i;
 425         struct progress *progress = NULL;
 426         struct pack_header *hdr = fill(sizeof(struct pack_header));
 427
 428         nr_objects = ntohl(hdr->hdr_entries);
 429
 430         if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
 431                 die("bad pack file");
 432         if (!pack_version_ok(hdr->hdr_version))
 433                 die("unknown pack file version %d", ntohl(hdr->hdr_version));
 434         use(sizeof(struct pack_header));
 435
 436         if (!quiet)
 437                 progress = start_progress("Unpacking objects", nr_objects);
 438         obj_list = xmalloc(nr_objects * sizeof(*obj_list));
 439         memset(obj_list, 0, nr_objects * sizeof(*obj_list));
 440         for (i = 0; i < nr_objects; i++) {
 441                 unpack_one(i);
 442                 display_progress(progress, i + 1);
 443         }
 444         stop_progress(&progress);
 445
 446         if (delta_list)
 447                 die("unresolved deltas left after unpacking");
 448 }
 449
 450 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
 451 {
 452         int i;
 453         unsigned char sha1[20];
 454
 455         git_config(git_default_config);
 456
 457         quiet = !isatty(2);
 458
 459         for (i = 1 ; i < argc; i++) {
 460                 const char *arg = argv[i];
 461
 462                 if (*arg == '-') {
 463                         if (!strcmp(arg, "-n")) {
 464                                 dry_run = 1;
 465                                 continue;
 466                         }
 467                         if (!strcmp(arg, "-q")) {
 468                                 quiet = 1;
 469                                 continue;
 470                         }
 471                         if (!strcmp(arg, "-r")) {
 472                                 recover = 1;
 473                                 continue;
 474                         }
 475                         if (!strcmp(arg, "--strict")) {
 476                                 strict = 1;
 477                                 continue;
 478                         }
 479                         if (!prefixcmp(arg, "--pack_header=")) {
 480                                 struct pack_header *hdr;
 481                                 char *c;
 482
 483                                 hdr = (struct pack_header *)buffer;
 484                                 hdr->hdr_signature = htonl(PACK_SIGNATURE);
 485                                 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
 486                                 if (*c != ',')
 487                                         die("bad %s", arg);
 488                                 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
 489                                 if (*c)
 490                                         die("bad %s", arg);
 491                                 len = sizeof(*hdr);
 492                                 continue;
 493                         }
 494                         usage(unpack_usage);
 495                 }
 496
 497                 /* We don't take any non-flag arguments now.. Maybe some day */
 498                 usage(unpack_usage);
 499         }
 500         SHA1_Init(&ctx);
 501         unpack_all();
 502         SHA1_Update(&ctx, buffer, offset);
 503         SHA1_Final(sha1, &ctx);
 504         if (strict)
 505                 write_rest();
 506         if (hashcmp(fill(20), sha1))
 507                 die("final sha1 did not match");
 508         use(20);
 509
 510         /* Write the last part of the buffer to stdout */
 511         while (len) {
 512                 int ret = xwrite(1, buffer + offset, len);
 513                 if (ret <= 0)
 514                         break;
 515                 len -= ret;
 516                 offset += ret;
 517         }
 518
 519         /* All done */
 520         return has_errors;
 521 }