X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=fast-import.c;h=f4bfe0f6650fdb2666f031ca1f551134f872e098;hb=5f4d133fca7eb3cb750f056a6fd0281ba06cd12f;hp=74f08bd554ae4cab1dc9ee38272376e5cbec1200;hpb=03f94ae9f909952ed5a78917ab319a312889354b;p=git.git diff --git a/fast-import.c b/fast-import.c index 74f08bd55..f4bfe0f66 100644 --- a/fast-import.c +++ b/fast-import.c @@ -24,10 +24,12 @@ Format of STDIN stream: commit_msg ('from' sp committish lf)? ('merge' sp committish lf)* - file_change* + (file_change | ls)* lf?; commit_msg ::= data; + ls ::= 'ls' sp '"' quoted(path) '"' lf; + file_change ::= file_clr | file_del | file_rnm @@ -132,14 +134,19 @@ Format of STDIN stream: ts ::= # time since the epoch in seconds, ascii base10 notation; tz ::= # GIT style timezone; - # note: comments may appear anywhere in the input, except - # within a data command. Any form of the data command - # always escapes the related input from comment processing. + # note: comments, ls and cat requests may appear anywhere + # in the input, except within a data command. Any form + # of the data command always escapes the related input + # from comment processing. # # In case it is not clear, the '#' that starts the comment # must be the first character on that line (an lf # preceded it). # + + cat_blob ::= 'cat-blob' sp (hexsha1 | idnum) lf; + ls_tree ::= 'ls' sp (hexsha1 | idnum) sp path_str lf; + comment ::= '#' not_lf* lf; not_lf ::= # Any byte that is not ASCII newline (LF); */ @@ -156,14 +163,19 @@ Format of STDIN stream: #include "csum-file.h" #include "quote.h" #include "exec_cmd.h" +#include "dir.h" #define PACK_ID_BITS 16 #define MAX_PACK_ID ((1<idx.sha1)) return e; - p = e; e = e->next; } e = new_object(sha1); - e->next = NULL; + e->next = object_table[h]; e->idx.offset = 0; - if (p) - p->next = e; - else - object_table[h] = e; + object_table[h] = e; return e; } @@ -691,13 +722,8 @@ static struct branch *new_branch(const char *name) if (b) die("Invalid attempt to create duplicate branch: %s", name); - switch (check_ref_format(name)) { - case 0: break; /* its valid */ - case CHECK_REF_FORMAT_ONELEVEL: - break; /* valid, but too few '/', allow anyway */ - default: + if (check_refname_format(name, REFNAME_ALLOW_ONELEVEL)) die("Branch name doesn't conform to GIT standards: %s", name); - } b = pool_calloc(1, sizeof(struct branch)); b->name = pool_strdup(name); @@ -839,6 +865,7 @@ static void start_packfile(void) p = xcalloc(1, sizeof(*p) + strlen(tmpfile) + 2); strcpy(p->pack_name, tmpfile); p->pack_fd = pack_fd; + p->do_not_close = 1; pack_file = sha1fd(pack_fd, p->pack_name); hdr.hdr_signature = htonl(PACK_SIGNATURE); @@ -872,7 +899,7 @@ static const char *create_index(void) if (c != last) die("internal consistency error creating the index"); - tmpfile = write_idx_file(NULL, idx, object_count, pack_data->sha1); + tmpfile = write_idx_file(NULL, idx, object_count, &pack_idx_opts, pack_data->sha1); free(idx); return tmpfile; } @@ -980,29 +1007,6 @@ static void cycle_packfile(void) start_packfile(); } -static size_t encode_header( - enum object_type type, - uintmax_t size, - unsigned char *hdr) -{ - int n = 1; - unsigned char c; - - if (type < OBJ_COMMIT || type > OBJ_REF_DELTA) - die("bad type %d", type); - - c = (type << 4) | (size & 15); - size >>= 4; - while (size) { - *hdr++ = c | 0x80; - c = size & 0x7f; - size >>= 7; - n++; - } - *hdr = c; - return n; -} - static int store_object( enum object_type type, struct strbuf *dat, @@ -1016,7 +1020,7 @@ static int store_object( unsigned char sha1[20]; unsigned long hdrlen, deltalen; git_SHA_CTX c; - z_stream s; + git_zstream s; hdrlen = sprintf((char *)hdr,"%s %lu", typename(type), (unsigned long)dat->len) + 1; @@ -1042,6 +1046,7 @@ static int store_object( } if (last && last->data.buf && last->depth < max_depth && dat->len > 20) { + delta_count_attempts_by_type[type]++; delta = diff_delta(last->data.buf, last->data.len, dat->buf, dat->len, &deltalen, dat->len - 20); @@ -1049,7 +1054,7 @@ static int store_object( delta = NULL; memset(&s, 0, sizeof(s)); - deflateInit(&s, pack_compression_level); + git_deflate_init(&s, pack_compression_level); if (delta) { s.next_in = delta; s.avail_in = deltalen; @@ -1057,11 +1062,11 @@ static int store_object( s.next_in = (void *)dat->buf; s.avail_in = dat->len; } - s.avail_out = deflateBound(&s, s.avail_in); + s.avail_out = git_deflate_bound(&s, s.avail_in); s.next_out = out = xmalloc(s.avail_out); - while (deflate(&s, Z_FINISH) == Z_OK) - /* nothing */; - deflateEnd(&s); + while (git_deflate(&s, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&s); /* Determine if we should auto-checkpoint. */ if ((max_packsize && (pack_size + 60 + s.total_out) > max_packsize) @@ -1077,14 +1082,14 @@ static int store_object( delta = NULL; memset(&s, 0, sizeof(s)); - deflateInit(&s, pack_compression_level); + git_deflate_init(&s, pack_compression_level); s.next_in = (void *)dat->buf; s.avail_in = dat->len; - s.avail_out = deflateBound(&s, s.avail_in); + s.avail_out = git_deflate_bound(&s, s.avail_in); s.next_out = out = xrealloc(out, s.avail_out); - while (deflate(&s, Z_FINISH) == Z_OK) - /* nothing */; - deflateEnd(&s); + while (git_deflate(&s, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&s); } } @@ -1103,7 +1108,7 @@ static int store_object( delta_count_by_type[type]++; e->depth = last->depth + 1; - hdrlen = encode_header(OBJ_OFS_DELTA, deltalen, hdr); + hdrlen = encode_in_pack_object_header(OBJ_OFS_DELTA, deltalen, hdr); sha1write(pack_file, hdr, hdrlen); pack_size += hdrlen; @@ -1114,7 +1119,7 @@ static int store_object( pack_size += sizeof(hdr) - pos; } else { e->depth = 0; - hdrlen = encode_header(type, dat->len, hdr); + hdrlen = encode_in_pack_object_header(type, dat->len, hdr); sha1write(pack_file, hdr, hdrlen); pack_size += hdrlen; } @@ -1162,7 +1167,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) off_t offset; git_SHA_CTX c; git_SHA_CTX pack_file_ctx; - z_stream s; + git_zstream s; int status = Z_OK; /* Determine if we should auto-checkpoint. */ @@ -1186,9 +1191,9 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) crc32_begin(pack_file); memset(&s, 0, sizeof(s)); - deflateInit(&s, pack_compression_level); + git_deflate_init(&s, pack_compression_level); - hdrlen = encode_header(OBJ_BLOB, len, out_buf); + hdrlen = encode_in_pack_object_header(OBJ_BLOB, len, out_buf); if (out_sz <= hdrlen) die("impossibly large object header"); @@ -1208,7 +1213,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) len -= n; } - status = deflate(&s, len ? 0 : Z_FINISH); + status = git_deflate(&s, len ? 0 : Z_FINISH); if (!s.avail_out || status == Z_STREAM_END) { size_t n = s.next_out - out_buf; @@ -1227,7 +1232,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) die("unexpected deflate failure: %d", status); } } - deflateEnd(&s); + git_deflate_end(&s); git_SHA1_Final(sha1, &c); if (sha1out) @@ -1413,8 +1418,9 @@ static void mktree(struct tree_content *t, int v, struct strbuf *b) struct tree_entry *e = t->entries[i]; if (!e->versions[v].mode) continue; - strbuf_addf(b, "%o %s%c", (unsigned int)e->versions[v].mode, - e->name->str_dat, '\0'); + strbuf_addf(b, "%o %s%c", + (unsigned int)(e->versions[v].mode & ~NO_DELTA), + e->name->str_dat, '\0'); strbuf_add(b, e->versions[v].sha1, 20); } } @@ -1424,7 +1430,7 @@ static void store_tree(struct tree_entry *root) struct tree_content *t = root->tree; unsigned int i, j, del; struct last_object lo = { STRBUF_INIT, 0, 0, /* no_swap */ 1 }; - struct object_entry *le; + struct object_entry *le = NULL; if (!is_null_sha1(root->versions[1].sha1)) return; @@ -1434,7 +1440,8 @@ static void store_tree(struct tree_entry *root) store_tree(t->entries[i]); } - le = find_object(root->versions[0].sha1); + if (!(root->versions[0].mode & NO_DELTA)) + le = find_object(root->versions[0].sha1); if (S_ISDIR(root->versions[0].mode) && le && le->pack_id == pack_id) { mktree(t, 0, &old_tree); lo.data = old_tree; @@ -1460,6 +1467,21 @@ static void store_tree(struct tree_entry *root) t->entry_count -= del; } +static void tree_content_replace( + struct tree_entry *root, + const unsigned char *sha1, + const uint16_t mode, + struct tree_content *newtree) +{ + if (!S_ISDIR(mode)) + die("Root cannot be a non-directory"); + hashclr(root->versions[0].sha1); + hashcpy(root->versions[1].sha1, sha1); + if (root->tree) + release_tree_content_recursive(root->tree); + root->tree = newtree; +} + static int tree_content_set( struct tree_entry *root, const char *p, @@ -1467,7 +1489,7 @@ static int tree_content_set( const uint16_t mode, struct tree_content *subtree) { - struct tree_content *t = root->tree; + struct tree_content *t; const char *slash1; unsigned int i, n; struct tree_entry *e; @@ -1482,9 +1504,12 @@ static int tree_content_set( if (!slash1 && !S_ISDIR(mode) && subtree) die("Non-directories cannot have subtrees"); + if (!root->tree) + load_tree(root); + t = root->tree; for (i = 0; i < t->entry_count; i++) { e = t->entries[i]; - if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) { + if (e->name->str_len == n && !strncmp_icase(p, e->name->str_dat, n)) { if (!slash1) { if (!S_ISDIR(mode) && e->versions[1].mode == mode @@ -1495,6 +1520,23 @@ static int tree_content_set( if (e->tree) release_tree_content_recursive(e->tree); e->tree = subtree; + + /* + * We need to leave e->versions[0].sha1 alone + * to avoid modifying the preimage tree used + * when writing out the parent directory. + * But after replacing the subdir with a + * completely different one, it's not a good + * delta base any more, and besides, we've + * thrown away the tree entries needed to + * make a delta against it. + * + * So let's just explicitly disable deltas + * for the subtree. + */ + if (S_ISDIR(e->versions[0].mode)) + e->versions[0].mode |= NO_DELTA; + hashclr(root->versions[1].sha1); return 1; } @@ -1537,7 +1579,7 @@ static int tree_content_remove( const char *p, struct tree_entry *backup_leaf) { - struct tree_content *t = root->tree; + struct tree_content *t; const char *slash1; unsigned int i, n; struct tree_entry *e; @@ -1548,9 +1590,20 @@ static int tree_content_remove( else n = strlen(p); + if (!root->tree) + load_tree(root); + t = root->tree; for (i = 0; i < t->entry_count; i++) { e = t->entries[i]; - if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) { + if (e->name->str_len == n && !strncmp_icase(p, e->name->str_dat, n)) { + if (slash1 && !S_ISDIR(e->versions[1].mode)) + /* + * If p names a file in some subdirectory, and a + * file or symlink matching the name of the + * parent directory of p exists, then p cannot + * exist and need not be deleted. + */ + return 1; if (!slash1 || !S_ISDIR(e->versions[1].mode)) goto del_entry; if (!e->tree) @@ -1587,7 +1640,7 @@ static int tree_content_get( const char *p, struct tree_entry *leaf) { - struct tree_content *t = root->tree; + struct tree_content *t; const char *slash1; unsigned int i, n; struct tree_entry *e; @@ -1598,9 +1651,12 @@ static int tree_content_get( else n = strlen(p); + if (!root->tree) + load_tree(root); + t = root->tree; for (i = 0; i < t->entry_count; i++) { e = t->entries[i]; - if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) { + if (e->name->str_len == n && !strncmp_icase(p, e->name->str_dat, n)) { if (!slash1) { memcpy(leaf, e, sizeof(*leaf)); if (e->tree && is_null_sha1(e->versions[1].sha1)) @@ -1689,7 +1745,7 @@ static void dump_marks_helper(FILE *f, if (m->shift) { for (k = 0; k < 1024; k++) { if (m->data.sets[k]) - dump_marks_helper(f, (base + k) << m->shift, + dump_marks_helper(f, base + (k << m->shift), m->data.sets[k]); } } else { @@ -1755,7 +1811,11 @@ static void read_marks(void) { char line[512]; FILE *f = fopen(import_marks_file, "r"); - if (!f) + if (f) + ; + else if (import_marks_file_ignore_missing && errno == ENOENT) + return; /* Marks file does not exist */ + else die_errno("cannot read '%s'", import_marks_file); while (fgets(line, sizeof(line), f)) { uintmax_t mark; @@ -1796,7 +1856,7 @@ static int read_next_command(void) return EOF; } - do { + for (;;) { if (unread_command_buf) { unread_command_buf = 0; } else { @@ -1829,9 +1889,14 @@ static int read_next_command(void) rc->prev->next = rc; cmd_tail = rc; } - } while (command_buf.buf[0] == '#'); - - return 0; + if (!prefixcmp(command_buf.buf, "cat-blob ")) { + parse_cat_blob(); + continue; + } + if (command_buf.buf[0] == '#') + continue; + return 0; + } } static void skip_optional_lf(void) @@ -1926,32 +1991,41 @@ static int validate_raw_date(const char *src, char *result, int maxlen) static char *parse_ident(const char *buf) { - const char *gt; + const char *ltgt; size_t name_len; char *ident; - gt = strrchr(buf, '>'); - if (!gt) + /* ensure there is a space delimiter even if there is no name */ + if (*buf == '<') + --buf; + + ltgt = buf + strcspn(buf, "<>"); + if (*ltgt != '<') + die("Missing < in ident string: %s", buf); + if (ltgt != buf && ltgt[-1] != ' ') + die("Missing space before < in ident string: %s", buf); + ltgt = ltgt + 1 + strcspn(ltgt + 1, "<>"); + if (*ltgt != '>') die("Missing > in ident string: %s", buf); - gt++; - if (*gt != ' ') + ltgt++; + if (*ltgt != ' ') die("Missing space after > in ident string: %s", buf); - gt++; - name_len = gt - buf; + ltgt++; + name_len = ltgt - buf; ident = xmalloc(name_len + 24); strncpy(ident, buf, name_len); switch (whenspec) { case WHENSPEC_RAW: - if (validate_raw_date(gt, ident + name_len, 24) < 0) - die("Invalid raw date \"%s\" in ident: %s", gt, buf); + if (validate_raw_date(ltgt, ident + name_len, 24) < 0) + die("Invalid raw date \"%s\" in ident: %s", ltgt, buf); break; case WHENSPEC_RFC2822: - if (parse_date(gt, ident + name_len, 24) < 0) - die("Invalid rfc2822 date \"%s\" in ident: %s", gt, buf); + if (parse_date(ltgt, ident + name_len, 24) < 0) + die("Invalid rfc2822 date \"%s\" in ident: %s", ltgt, buf); break; case WHENSPEC_NOW: - if (strcmp("now", gt)) + if (strcmp("now", ltgt)) die("Date in ident must be 'now': %s", buf); datestamp(ident + name_len, 24); break; @@ -2099,6 +2173,11 @@ static uintmax_t do_change_note_fanout( if (tmp_hex_sha1_len == 40 && !get_sha1_hex(hex_sha1, sha1)) { /* This is a note entry */ + if (fanout == 0xff) { + /* Counting mode, no rename */ + num_notes++; + continue; + } construct_path_with_fanout(hex_sha1, fanout, realpath); if (!strcmp(fullpath, realpath)) { /* Note entry is in correct location */ @@ -2154,6 +2233,7 @@ static void file_change_m(struct branch *b) case S_IFREG | 0644: case S_IFREG | 0755: case S_IFLNK: + case S_IFDIR: case S_IFGITLINK: /* ok */ break; @@ -2185,6 +2265,12 @@ static void file_change_m(struct branch *b) p = uq.buf; } + /* Git does not track empty, non-toplevel directories. */ + if (S_ISDIR(mode) && !memcmp(sha1, EMPTY_TREE_SHA1_BIN, 20) && *p) { + tree_content_remove(&b->branch_tree, p, NULL); + return; + } + if (S_ISGITLINK(mode)) { if (inline_data) die("Git links cannot be specified 'inline': %s", @@ -2199,25 +2285,34 @@ static void file_change_m(struct branch *b) * another repository. */ } else if (inline_data) { + if (S_ISDIR(mode)) + die("Directories cannot be specified 'inline': %s", + command_buf.buf); if (p != uq.buf) { strbuf_addstr(&uq, p); p = uq.buf; } read_next_command(); parse_and_store_blob(&last_blob, sha1, 0); - } else if (oe) { - if (oe->type != OBJ_BLOB) - die("Not a blob (actually a %s): %s", - typename(oe->type), command_buf.buf); } else { - enum object_type type = sha1_object_info(sha1, NULL); + enum object_type expected = S_ISDIR(mode) ? + OBJ_TREE: OBJ_BLOB; + enum object_type type = oe ? oe->type : + sha1_object_info(sha1, NULL); if (type < 0) - die("Blob not found: %s", command_buf.buf); - if (type != OBJ_BLOB) - die("Not a blob (actually a %s): %s", - typename(type), command_buf.buf); + die("%s not found: %s", + S_ISDIR(mode) ? "Tree" : "Blob", + command_buf.buf); + if (type != expected) + die("Not a %s (actually a %s): %s", + typename(expected), typename(type), + command_buf.buf); } + if (!*p) { + tree_content_replace(&b->branch_tree, sha1, mode, NULL); + return; + } tree_content_set(&b->branch_tree, p, sha1, mode, NULL); } @@ -2276,13 +2371,20 @@ static void file_change_cr(struct branch *b, int rename) tree_content_get(&b->branch_tree, s, &leaf); if (!leaf.versions[1].mode) die("Path %s not in branch", s); + if (!*d) { /* C "path/to/subdir" "" */ + tree_content_replace(&b->branch_tree, + leaf.versions[1].sha1, + leaf.versions[1].mode, + leaf.tree); + return; + } tree_content_set(&b->branch_tree, d, leaf.versions[1].sha1, leaf.versions[1].mode, leaf.tree); } -static void note_change_n(struct branch *b, unsigned char old_fanout) +static void note_change_n(struct branch *b, unsigned char *old_fanout) { const char *p = command_buf.buf + 2; static struct strbuf uq = STRBUF_INIT; @@ -2293,6 +2395,23 @@ static void note_change_n(struct branch *b, unsigned char old_fanout) uint16_t inline_data = 0; unsigned char new_fanout; + /* + * When loading a branch, we don't traverse its tree to count the real + * number of notes (too expensive to do this for all non-note refs). + * This means that recently loaded notes refs might incorrectly have + * b->num_notes == 0, and consequently, old_fanout might be wrong. + * + * Fix this by traversing the tree and counting the number of notes + * when b->num_notes == 0. If the notes tree is truly empty, the + * calculation should not take long. + */ + if (b->num_notes == 0 && *old_fanout == 0) { + /* Invoke change_note_fanout() in "counting mode". */ + b->num_notes = change_note_fanout(&b->branch_tree, 0xff); + *old_fanout = convert_num_notes_to_fanout(b->num_notes); + } + + /* Now parse the notemodify command. */ /* or 'inline' */ if (*p == ':') { char *x; @@ -2314,6 +2433,8 @@ static void note_change_n(struct branch *b, unsigned char old_fanout) /* */ s = lookup_branch(p); if (s) { + if (is_null_sha1(s->sha1)) + die("Can't add a note on empty branch."); hashcpy(commit_sha1, s->sha1); } else if (*p == ':') { uintmax_t commit_mark = strtoumax(p + 1, NULL, 10); @@ -2351,7 +2472,7 @@ static void note_change_n(struct branch *b, unsigned char old_fanout) typename(type), command_buf.buf); } - construct_path_with_fanout(sha1_to_hex(commit_sha1), old_fanout, path); + construct_path_with_fanout(sha1_to_hex(commit_sha1), *old_fanout, path); if (tree_content_remove(&b->branch_tree, path, NULL)) b->num_notes--; @@ -2538,9 +2659,11 @@ static void parse_new_commit(void) else if (!prefixcmp(command_buf.buf, "C ")) file_change_cr(b, 0); else if (!prefixcmp(command_buf.buf, "N ")) - note_change_n(b, prev_fanout); + note_change_n(b, &prev_fanout); else if (!strcmp("deleteall", command_buf.buf)) file_change_deleteall(b); + else if (!prefixcmp(command_buf.buf, "ls ")) + parse_ls(b); else { unread_command_buf = 1; break; @@ -2613,6 +2736,8 @@ static void parse_new_tag(void) from = strchr(command_buf.buf, ' ') + 1; s = lookup_branch(from); if (s) { + if (is_null_sha1(s->sha1)) + die("Can't tag an empty branch."); hashcpy(sha1, s->sha1); type = OBJ_COMMIT; } else if (*from == ':') { @@ -2622,13 +2747,13 @@ static void parse_new_tag(void) type = oe->type; hashcpy(sha1, oe->idx.sha1); } else if (!get_sha1(from, sha1)) { - unsigned long size; - char *buf; - - buf = read_sha1_file(sha1, &type, &size); - if (!buf || size < 46) - die("Not a valid commit: %s", from); - free(buf); + struct object_entry *oe = find_object(sha1); + if (!oe) { + type = sha1_object_info(sha1, NULL); + if (type < 0) + die("Not a valid object: %s", from); + } else + type = oe->type; } else die("Invalid ref name or SHA1 expression: %s", from); read_next_command(); @@ -2689,14 +2814,247 @@ static void parse_reset_branch(void) unread_command_buf = 1; } -static void parse_checkpoint(void) +static void cat_blob_write(const char *buf, unsigned long size) +{ + if (write_in_full(cat_blob_fd, buf, size) != size) + die_errno("Write to frontend failed"); +} + +static void cat_blob(struct object_entry *oe, unsigned char sha1[20]) +{ + struct strbuf line = STRBUF_INIT; + unsigned long size; + enum object_type type = 0; + char *buf; + + if (!oe || oe->pack_id == MAX_PACK_ID) { + buf = read_sha1_file(sha1, &type, &size); + } else { + type = oe->type; + buf = gfi_unpack_entry(oe, &size); + } + + /* + * Output based on batch_one_object() from cat-file.c. + */ + if (type <= 0) { + strbuf_reset(&line); + strbuf_addf(&line, "%s missing\n", sha1_to_hex(sha1)); + cat_blob_write(line.buf, line.len); + strbuf_release(&line); + free(buf); + return; + } + if (!buf) + die("Can't read object %s", sha1_to_hex(sha1)); + if (type != OBJ_BLOB) + die("Object %s is a %s but a blob was expected.", + sha1_to_hex(sha1), typename(type)); + strbuf_reset(&line); + strbuf_addf(&line, "%s %s %lu\n", sha1_to_hex(sha1), + typename(type), size); + cat_blob_write(line.buf, line.len); + strbuf_release(&line); + cat_blob_write(buf, size); + cat_blob_write("\n", 1); + if (oe && oe->pack_id == pack_id) { + last_blob.offset = oe->idx.offset; + strbuf_attach(&last_blob.data, buf, size, size); + last_blob.depth = oe->depth; + } else + free(buf); +} + +static void parse_cat_blob(void) +{ + const char *p; + struct object_entry *oe = oe; + unsigned char sha1[20]; + + /* cat-blob SP LF */ + p = command_buf.buf + strlen("cat-blob "); + if (*p == ':') { + char *x; + oe = find_mark(strtoumax(p + 1, &x, 10)); + if (x == p + 1) + die("Invalid mark: %s", command_buf.buf); + if (!oe) + die("Unknown mark: %s", command_buf.buf); + if (*x) + die("Garbage after mark: %s", command_buf.buf); + hashcpy(sha1, oe->idx.sha1); + } else { + if (get_sha1_hex(p, sha1)) + die("Invalid SHA1: %s", command_buf.buf); + if (p[40]) + die("Garbage after SHA1: %s", command_buf.buf); + oe = find_object(sha1); + } + + cat_blob(oe, sha1); +} + +static struct object_entry *dereference(struct object_entry *oe, + unsigned char sha1[20]) +{ + unsigned long size; + char *buf = NULL; + if (!oe) { + enum object_type type = sha1_object_info(sha1, NULL); + if (type < 0) + die("object not found: %s", sha1_to_hex(sha1)); + /* cache it! */ + oe = insert_object(sha1); + oe->type = type; + oe->pack_id = MAX_PACK_ID; + oe->idx.offset = 1; + } + switch (oe->type) { + case OBJ_TREE: /* easy case. */ + return oe; + case OBJ_COMMIT: + case OBJ_TAG: + break; + default: + die("Not a treeish: %s", command_buf.buf); + } + + if (oe->pack_id != MAX_PACK_ID) { /* in a pack being written */ + buf = gfi_unpack_entry(oe, &size); + } else { + enum object_type unused; + buf = read_sha1_file(sha1, &unused, &size); + } + if (!buf) + die("Can't load object %s", sha1_to_hex(sha1)); + + /* Peel one layer. */ + switch (oe->type) { + case OBJ_TAG: + if (size < 40 + strlen("object ") || + get_sha1_hex(buf + strlen("object "), sha1)) + die("Invalid SHA1 in tag: %s", command_buf.buf); + break; + case OBJ_COMMIT: + if (size < 40 + strlen("tree ") || + get_sha1_hex(buf + strlen("tree "), sha1)) + die("Invalid SHA1 in commit: %s", command_buf.buf); + } + + free(buf); + return find_object(sha1); +} + +static struct object_entry *parse_treeish_dataref(const char **p) +{ + unsigned char sha1[20]; + struct object_entry *e; + + if (**p == ':') { /* */ + char *endptr; + e = find_mark(strtoumax(*p + 1, &endptr, 10)); + if (endptr == *p + 1) + die("Invalid mark: %s", command_buf.buf); + if (!e) + die("Unknown mark: %s", command_buf.buf); + *p = endptr; + hashcpy(sha1, e->idx.sha1); + } else { /* */ + if (get_sha1_hex(*p, sha1)) + die("Invalid SHA1: %s", command_buf.buf); + e = find_object(sha1); + *p += 40; + } + + while (!e || e->type != OBJ_TREE) + e = dereference(e, sha1); + return e; +} + +static void print_ls(int mode, const unsigned char *sha1, const char *path) +{ + static struct strbuf line = STRBUF_INIT; + + /* See show_tree(). */ + const char *type = + S_ISGITLINK(mode) ? commit_type : + S_ISDIR(mode) ? tree_type : + blob_type; + + if (!mode) { + /* missing SP path LF */ + strbuf_reset(&line); + strbuf_addstr(&line, "missing "); + quote_c_style(path, &line, NULL, 0); + strbuf_addch(&line, '\n'); + } else { + /* mode SP type SP object_name TAB path LF */ + strbuf_reset(&line); + strbuf_addf(&line, "%06o %s %s\t", + mode & ~NO_DELTA, type, sha1_to_hex(sha1)); + quote_c_style(path, &line, NULL, 0); + strbuf_addch(&line, '\n'); + } + cat_blob_write(line.buf, line.len); +} + +static void parse_ls(struct branch *b) +{ + const char *p; + struct tree_entry *root = NULL; + struct tree_entry leaf = {NULL}; + + /* ls SP ( SP)? */ + p = command_buf.buf + strlen("ls "); + if (*p == '"') { + if (!b) + die("Not in a commit: %s", command_buf.buf); + root = &b->branch_tree; + } else { + struct object_entry *e = parse_treeish_dataref(&p); + root = new_tree_entry(); + hashcpy(root->versions[1].sha1, e->idx.sha1); + load_tree(root); + if (*p++ != ' ') + die("Missing space after tree-ish: %s", command_buf.buf); + } + if (*p == '"') { + static struct strbuf uq = STRBUF_INIT; + const char *endp; + strbuf_reset(&uq); + if (unquote_c_style(&uq, p, &endp)) + die("Invalid path: %s", command_buf.buf); + if (*endp) + die("Garbage after path in: %s", command_buf.buf); + p = uq.buf; + } + tree_content_get(root, p, &leaf); + /* + * A directory in preparation would have a sha1 of zero + * until it is saved. Save, for simplicity. + */ + if (S_ISDIR(leaf.versions[1].mode)) + store_tree(&leaf); + + print_ls(leaf.versions[1].mode, leaf.versions[1].sha1, p); + if (!b || root != &b->branch_tree) + release_tree_entry(root); +} + +static void checkpoint(void) { + checkpoint_requested = 0; if (object_count) { cycle_packfile(); dump_branches(); dump_tags(); dump_marks(); } +} + +static void parse_checkpoint(void) +{ + checkpoint_requested = 1; skip_optional_lf(); } @@ -2718,7 +3076,8 @@ static char* make_fast_import_path(const char *path) return strbuf_detach(&abs_path, NULL); } -static void option_import_marks(const char *marks, int from_stream) +static void option_import_marks(const char *marks, + int from_stream, int ignore_missing) { if (import_marks_file) { if (from_stream) @@ -2730,7 +3089,9 @@ static void option_import_marks(const char *marks, int from_stream) } import_marks_file = make_fast_import_path(marks); + safe_create_leading_directories_const(import_marks_file); import_marks_file_from_stream = from_stream; + import_marks_file_ignore_missing = ignore_missing; } static void option_date_format(const char *fmt) @@ -2745,21 +3106,39 @@ static void option_date_format(const char *fmt) die("unknown --date-format argument %s", fmt); } +static unsigned long ulong_arg(const char *option, const char *arg) +{ + char *endptr; + unsigned long rv = strtoul(arg, &endptr, 0); + if (strchr(arg, '-') || endptr == arg || *endptr) + die("%s: argument must be a non-negative integer", option); + return rv; +} + static void option_depth(const char *depth) { - max_depth = strtoul(depth, NULL, 0); + max_depth = ulong_arg("--depth", depth); if (max_depth > MAX_DEPTH) die("--depth cannot exceed %u", MAX_DEPTH); } static void option_active_branches(const char *branches) { - max_active_branches = strtoul(branches, NULL, 0); + max_active_branches = ulong_arg("--active-branches", branches); } static void option_export_marks(const char *marks) { export_marks_file = make_fast_import_path(marks); + safe_create_leading_directories_const(export_marks_file); +} + +static void option_cat_blob_fd(const char *fd) +{ + unsigned long n = ulong_arg("--cat-blob-fd", fd); + if (n > (unsigned long) INT_MAX) + die("--cat-blob-fd cannot exceed %d", INT_MAX); + cat_blob_fd = (int) n; } static void option_export_pack_edges(const char *edges) @@ -2812,15 +3191,24 @@ static int parse_one_feature(const char *feature, int from_stream) if (!prefixcmp(feature, "date-format=")) { option_date_format(feature + 12); } else if (!prefixcmp(feature, "import-marks=")) { - option_import_marks(feature + 13, from_stream); + option_import_marks(feature + 13, from_stream, 0); + } else if (!prefixcmp(feature, "import-marks-if-exists=")) { + option_import_marks(feature + strlen("import-marks-if-exists="), + from_stream, 1); } else if (!prefixcmp(feature, "export-marks=")) { option_export_marks(feature + 13); - } else if (!prefixcmp(feature, "relative-marks")) { + } else if (!strcmp(feature, "cat-blob")) { + ; /* Don't die - this feature is supported */ + } else if (!strcmp(feature, "relative-marks")) { relative_marks_paths = 1; - } else if (!prefixcmp(feature, "no-relative-marks")) { + } else if (!strcmp(feature, "no-relative-marks")) { relative_marks_paths = 0; - } else if (!prefixcmp(feature, "force")) { + } else if (!strcmp(feature, "done")) { + require_explicit_termination = 1; + } else if (!strcmp(feature, "force")) { force_update = 1; + } else if (!strcmp(feature, "notes") || !strcmp(feature, "ls")) { + ; /* do nothing; we have the feature */ } else { return 0; } @@ -2873,25 +3261,21 @@ static int git_pack_config(const char *k, const char *v, void *cb) return 0; } if (!strcmp(k, "pack.indexversion")) { - pack_idx_default_version = git_config_int(k, v); - if (pack_idx_default_version > 2) + pack_idx_opts.version = git_config_int(k, v); + if (pack_idx_opts.version > 2) die("bad pack.indexversion=%"PRIu32, - pack_idx_default_version); + pack_idx_opts.version); return 0; } if (!strcmp(k, "pack.packsizelimit")) { max_packsize = git_config_ulong(k, v); return 0; } - if (!strcmp(k, "core.bigfilethreshold")) { - long n = git_config_int(k, v); - big_file_threshold = 0 < n ? n : 0; - } return git_default_config(k, v, cb); } static const char fast_import_usage[] = -"git fast-import [--date-format=f] [--max-pack-size=n] [--big-file-threshold=n] [--depth=n] [--active-branches=n] [--export-marks=marks.file]"; +"git fast-import [--date-format=] [--max-pack-size=] [--big-file-threshold=] [--depth=] [--active-branches=] [--export-marks=]"; static void parse_argv(void) { @@ -2909,6 +3293,11 @@ static void parse_argv(void) if (parse_one_feature(a + 2, 0)) continue; + if (!prefixcmp(a + 2, "cat-blob-fd=")) { + option_cat_blob_fd(a + 2 + strlen("cat-blob-fd=")); + continue; + } + die("unknown option %s", a); } if (i != global_argc) @@ -2929,6 +3318,7 @@ int main(int argc, const char **argv) usage(fast_import_usage); setup_git_directory(); + reset_pack_idx_option(&pack_idx_opts); git_config(git_pack_config, NULL); if (!pack_compression_seen && core_compression_seen) pack_compression_level = core_compression_level; @@ -2951,9 +3341,12 @@ int main(int argc, const char **argv) prepare_packed_git(); start_packfile(); set_die_routine(die_nicely); + set_checkpoint_signal(); while (read_next_command() != EOF) { if (!strcmp("blob", command_buf.buf)) parse_new_blob(); + else if (!prefixcmp(command_buf.buf, "ls ")) + parse_ls(NULL); else if (!prefixcmp(command_buf.buf, "commit ")) parse_new_commit(); else if (!prefixcmp(command_buf.buf, "tag ")) @@ -2962,6 +3355,8 @@ int main(int argc, const char **argv) parse_reset_branch(); else if (!strcmp("checkpoint", command_buf.buf)) parse_checkpoint(); + else if (!strcmp("done", command_buf.buf)) + break; else if (!prefixcmp(command_buf.buf, "progress ")) parse_progress(); else if (!prefixcmp(command_buf.buf, "feature ")) @@ -2972,12 +3367,18 @@ int main(int argc, const char **argv) /* ignore non-git options*/; else die("Unsupported command: %s", command_buf.buf); + + if (checkpoint_requested) + checkpoint(); } /* argv hasn't been parsed yet, do so */ if (!seen_data_command) parse_argv(); + if (require_explicit_termination && feof(stdin)) + die("stream ends early"); + end_packfile(); dump_branches(); @@ -2999,10 +3400,10 @@ int main(int argc, const char **argv) fprintf(stderr, "---------------------------------------------------------------------\n"); fprintf(stderr, "Alloc'd objects: %10" PRIuMAX "\n", alloc_count); fprintf(stderr, "Total objects: %10" PRIuMAX " (%10" PRIuMAX " duplicates )\n", total_count, duplicate_count); - fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]); - fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]); - fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]); - fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]); + fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB], delta_count_attempts_by_type[OBJ_BLOB]); + fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE], delta_count_attempts_by_type[OBJ_TREE]); + fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT], delta_count_attempts_by_type[OBJ_COMMIT]); + fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG], delta_count_attempts_by_type[OBJ_TAG]); fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count); fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count); fprintf(stderr, " atoms: %10u\n", atom_cnt);