X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=fast-import.c;h=8d8ea3c45c0be5481c7b452c27ee0d163d69fb00;hb=97f261b1e7f1af54aed700e61cdb8af74dbd2459;hp=b3ba778924908946a0e20ebfdae9bc2058e8a55e;hpb=ed36a48e6d246f4f60d44b27e8c1e660151cd0b4;p=git.git diff --git a/fast-import.c b/fast-import.c index b3ba77892..8d8ea3c45 100644 --- a/fast-import.c +++ b/fast-import.c @@ -24,10 +24,12 @@ Format of STDIN stream: commit_msg ('from' sp committish lf)? ('merge' sp committish lf)* - file_change* + (file_change | ls)* lf?; commit_msg ::= data; + ls ::= 'ls' sp '"' quoted(path) '"' lf; + file_change ::= file_clr | file_del | file_rnm @@ -132,7 +134,7 @@ Format of STDIN stream: ts ::= # time since the epoch in seconds, ascii base10 notation; tz ::= # GIT style timezone; - # note: comments and cat requests may appear anywhere + # note: comments, ls and cat requests may appear anywhere # in the input, except within a data command. Any form # of the data command always escapes the related input # from comment processing. @@ -141,7 +143,9 @@ Format of STDIN stream: # must be the first character on that line (an lf # preceded it). # + cat_blob ::= 'cat-blob' sp (hexsha1 | idnum) lf; + ls_tree ::= 'ls' sp (hexsha1 | idnum) sp path_str lf; comment ::= '#' not_lf* lf; not_lf ::= # Any byte that is not ASCII newline (LF); @@ -166,6 +170,11 @@ Format of STDIN stream: #define DEPTH_BITS 13 #define MAX_DEPTH ((1<name = pool_strdup(name); @@ -891,7 +899,7 @@ static const char *create_index(void) if (c != last) die("internal consistency error creating the index"); - tmpfile = write_idx_file(NULL, idx, object_count, pack_data->sha1); + tmpfile = write_idx_file(NULL, idx, object_count, &pack_idx_opts, pack_data->sha1); free(idx); return tmpfile; } @@ -1012,7 +1020,7 @@ static int store_object( unsigned char sha1[20]; unsigned long hdrlen, deltalen; git_SHA_CTX c; - z_stream s; + git_zstream s; hdrlen = sprintf((char *)hdr,"%s %lu", typename(type), (unsigned long)dat->len) + 1; @@ -1038,6 +1046,7 @@ static int store_object( } if (last && last->data.buf && last->depth < max_depth && dat->len > 20) { + delta_count_attempts_by_type[type]++; delta = diff_delta(last->data.buf, last->data.len, dat->buf, dat->len, &deltalen, dat->len - 20); @@ -1045,7 +1054,7 @@ static int store_object( delta = NULL; memset(&s, 0, sizeof(s)); - deflateInit(&s, pack_compression_level); + git_deflate_init(&s, pack_compression_level); if (delta) { s.next_in = delta; s.avail_in = deltalen; @@ -1053,11 +1062,11 @@ static int store_object( s.next_in = (void *)dat->buf; s.avail_in = dat->len; } - s.avail_out = deflateBound(&s, s.avail_in); + s.avail_out = git_deflate_bound(&s, s.avail_in); s.next_out = out = xmalloc(s.avail_out); - while (deflate(&s, Z_FINISH) == Z_OK) - /* nothing */; - deflateEnd(&s); + while (git_deflate(&s, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&s); /* Determine if we should auto-checkpoint. */ if ((max_packsize && (pack_size + 60 + s.total_out) > max_packsize) @@ -1073,14 +1082,14 @@ static int store_object( delta = NULL; memset(&s, 0, sizeof(s)); - deflateInit(&s, pack_compression_level); + git_deflate_init(&s, pack_compression_level); s.next_in = (void *)dat->buf; s.avail_in = dat->len; - s.avail_out = deflateBound(&s, s.avail_in); + s.avail_out = git_deflate_bound(&s, s.avail_in); s.next_out = out = xrealloc(out, s.avail_out); - while (deflate(&s, Z_FINISH) == Z_OK) - /* nothing */; - deflateEnd(&s); + while (git_deflate(&s, Z_FINISH) == Z_OK) + ; /* nothing */ + git_deflate_end(&s); } } @@ -1158,7 +1167,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) off_t offset; git_SHA_CTX c; git_SHA_CTX pack_file_ctx; - z_stream s; + git_zstream s; int status = Z_OK; /* Determine if we should auto-checkpoint. */ @@ -1182,7 +1191,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) crc32_begin(pack_file); memset(&s, 0, sizeof(s)); - deflateInit(&s, pack_compression_level); + git_deflate_init(&s, pack_compression_level); hdrlen = encode_in_pack_object_header(OBJ_BLOB, len, out_buf); if (out_sz <= hdrlen) @@ -1204,7 +1213,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) len -= n; } - status = deflate(&s, len ? 0 : Z_FINISH); + status = git_deflate(&s, len ? 0 : Z_FINISH); if (!s.avail_out || status == Z_STREAM_END) { size_t n = s.next_out - out_buf; @@ -1223,7 +1232,7 @@ static void stream_blob(uintmax_t len, unsigned char *sha1out, uintmax_t mark) die("unexpected deflate failure: %d", status); } } - deflateEnd(&s); + git_deflate_end(&s); git_SHA1_Final(sha1, &c); if (sha1out) @@ -1409,8 +1418,9 @@ static void mktree(struct tree_content *t, int v, struct strbuf *b) struct tree_entry *e = t->entries[i]; if (!e->versions[v].mode) continue; - strbuf_addf(b, "%o %s%c", (unsigned int)e->versions[v].mode, - e->name->str_dat, '\0'); + strbuf_addf(b, "%o %s%c", + (unsigned int)(e->versions[v].mode & ~NO_DELTA), + e->name->str_dat, '\0'); strbuf_add(b, e->versions[v].sha1, 20); } } @@ -1420,7 +1430,7 @@ static void store_tree(struct tree_entry *root) struct tree_content *t = root->tree; unsigned int i, j, del; struct last_object lo = { STRBUF_INIT, 0, 0, /* no_swap */ 1 }; - struct object_entry *le; + struct object_entry *le = NULL; if (!is_null_sha1(root->versions[1].sha1)) return; @@ -1430,7 +1440,8 @@ static void store_tree(struct tree_entry *root) store_tree(t->entries[i]); } - le = find_object(root->versions[0].sha1); + if (!(root->versions[0].mode & NO_DELTA)) + le = find_object(root->versions[0].sha1); if (S_ISDIR(root->versions[0].mode) && le && le->pack_id == pack_id) { mktree(t, 0, &old_tree); lo.data = old_tree; @@ -1464,6 +1475,7 @@ static void tree_content_replace( { if (!S_ISDIR(mode)) die("Root cannot be a non-directory"); + hashclr(root->versions[0].sha1); hashcpy(root->versions[1].sha1, sha1); if (root->tree) release_tree_content_recursive(root->tree); @@ -1508,6 +1520,23 @@ static int tree_content_set( if (e->tree) release_tree_content_recursive(e->tree); e->tree = subtree; + + /* + * We need to leave e->versions[0].sha1 alone + * to avoid modifying the preimage tree used + * when writing out the parent directory. + * But after replacing the subdir with a + * completely different one, it's not a good + * delta base any more, and besides, we've + * thrown away the tree entries needed to + * make a delta against it. + * + * So let's just explicitly disable deltas + * for the subtree. + */ + if (S_ISDIR(e->versions[0].mode)) + e->versions[0].mode |= NO_DELTA; + hashclr(root->versions[1].sha1); return 1; } @@ -1782,7 +1811,11 @@ static void read_marks(void) { char line[512]; FILE *f = fopen(import_marks_file, "r"); - if (!f) + if (f) + ; + else if (import_marks_file_ignore_missing && errno == ENOENT) + return; /* Marks file does not exist */ + else die_errno("cannot read '%s'", import_marks_file); while (fgets(line, sizeof(line), f)) { uintmax_t mark; @@ -1958,32 +1991,41 @@ static int validate_raw_date(const char *src, char *result, int maxlen) static char *parse_ident(const char *buf) { - const char *gt; + const char *ltgt; size_t name_len; char *ident; - gt = strrchr(buf, '>'); - if (!gt) + /* ensure there is a space delimiter even if there is no name */ + if (*buf == '<') + --buf; + + ltgt = buf + strcspn(buf, "<>"); + if (*ltgt != '<') + die("Missing < in ident string: %s", buf); + if (ltgt != buf && ltgt[-1] != ' ') + die("Missing space before < in ident string: %s", buf); + ltgt = ltgt + 1 + strcspn(ltgt + 1, "<>"); + if (*ltgt != '>') die("Missing > in ident string: %s", buf); - gt++; - if (*gt != ' ') + ltgt++; + if (*ltgt != ' ') die("Missing space after > in ident string: %s", buf); - gt++; - name_len = gt - buf; + ltgt++; + name_len = ltgt - buf; ident = xmalloc(name_len + 24); strncpy(ident, buf, name_len); switch (whenspec) { case WHENSPEC_RAW: - if (validate_raw_date(gt, ident + name_len, 24) < 0) - die("Invalid raw date \"%s\" in ident: %s", gt, buf); + if (validate_raw_date(ltgt, ident + name_len, 24) < 0) + die("Invalid raw date \"%s\" in ident: %s", ltgt, buf); break; case WHENSPEC_RFC2822: - if (parse_date(gt, ident + name_len, 24) < 0) - die("Invalid rfc2822 date \"%s\" in ident: %s", gt, buf); + if (parse_date(ltgt, ident + name_len, 24) < 0) + die("Invalid rfc2822 date \"%s\" in ident: %s", ltgt, buf); break; case WHENSPEC_NOW: - if (strcmp("now", gt)) + if (strcmp("now", ltgt)) die("Date in ident must be 'now': %s", buf); datestamp(ident + name_len, 24); break; @@ -2369,6 +2411,8 @@ static void note_change_n(struct branch *b, unsigned char old_fanout) /* */ s = lookup_branch(p); if (s) { + if (is_null_sha1(s->sha1)) + die("Can't add a note on empty branch."); hashcpy(commit_sha1, s->sha1); } else if (*p == ':') { uintmax_t commit_mark = strtoumax(p + 1, NULL, 10); @@ -2596,6 +2640,8 @@ static void parse_new_commit(void) note_change_n(b, prev_fanout); else if (!strcmp("deleteall", command_buf.buf)) file_change_deleteall(b); + else if (!prefixcmp(command_buf.buf, "ls ")) + parse_ls(b); else { unread_command_buf = 1; break; @@ -2668,6 +2714,8 @@ static void parse_new_tag(void) from = strchr(command_buf.buf, ' ') + 1; s = lookup_branch(from); if (s) { + if (is_null_sha1(s->sha1)) + die("Can't tag an empty branch."); hashcpy(sha1, s->sha1); type = OBJ_COMMIT; } else if (*from == ':') { @@ -2677,13 +2725,13 @@ static void parse_new_tag(void) type = oe->type; hashcpy(sha1, oe->idx.sha1); } else if (!get_sha1(from, sha1)) { - unsigned long size; - char *buf; - - buf = read_sha1_file(sha1, &type, &size); - if (!buf || size < 46) - die("Not a valid commit: %s", from); - free(buf); + struct object_entry *oe = find_object(sha1); + if (!oe) { + type = sha1_object_info(sha1, NULL); + if (type < 0) + die("Not a valid object: %s", from); + } else + type = oe->type; } else die("Invalid ref name or SHA1 expression: %s", from); read_next_command(); @@ -2787,7 +2835,12 @@ static void cat_blob(struct object_entry *oe, unsigned char sha1[20]) strbuf_release(&line); cat_blob_write(buf, size); cat_blob_write("\n", 1); - free(buf); + if (oe && oe->pack_id == pack_id) { + last_blob.offset = oe->idx.offset; + strbuf_attach(&last_blob.data, buf, size, size); + last_blob.depth = oe->depth; + } else + free(buf); } static void parse_cat_blob(void) @@ -2819,6 +2872,153 @@ static void parse_cat_blob(void) cat_blob(oe, sha1); } +static struct object_entry *dereference(struct object_entry *oe, + unsigned char sha1[20]) +{ + unsigned long size; + char *buf = NULL; + if (!oe) { + enum object_type type = sha1_object_info(sha1, NULL); + if (type < 0) + die("object not found: %s", sha1_to_hex(sha1)); + /* cache it! */ + oe = insert_object(sha1); + oe->type = type; + oe->pack_id = MAX_PACK_ID; + oe->idx.offset = 1; + } + switch (oe->type) { + case OBJ_TREE: /* easy case. */ + return oe; + case OBJ_COMMIT: + case OBJ_TAG: + break; + default: + die("Not a treeish: %s", command_buf.buf); + } + + if (oe->pack_id != MAX_PACK_ID) { /* in a pack being written */ + buf = gfi_unpack_entry(oe, &size); + } else { + enum object_type unused; + buf = read_sha1_file(sha1, &unused, &size); + } + if (!buf) + die("Can't load object %s", sha1_to_hex(sha1)); + + /* Peel one layer. */ + switch (oe->type) { + case OBJ_TAG: + if (size < 40 + strlen("object ") || + get_sha1_hex(buf + strlen("object "), sha1)) + die("Invalid SHA1 in tag: %s", command_buf.buf); + break; + case OBJ_COMMIT: + if (size < 40 + strlen("tree ") || + get_sha1_hex(buf + strlen("tree "), sha1)) + die("Invalid SHA1 in commit: %s", command_buf.buf); + } + + free(buf); + return find_object(sha1); +} + +static struct object_entry *parse_treeish_dataref(const char **p) +{ + unsigned char sha1[20]; + struct object_entry *e; + + if (**p == ':') { /* */ + char *endptr; + e = find_mark(strtoumax(*p + 1, &endptr, 10)); + if (endptr == *p + 1) + die("Invalid mark: %s", command_buf.buf); + if (!e) + die("Unknown mark: %s", command_buf.buf); + *p = endptr; + hashcpy(sha1, e->idx.sha1); + } else { /* */ + if (get_sha1_hex(*p, sha1)) + die("Invalid SHA1: %s", command_buf.buf); + e = find_object(sha1); + *p += 40; + } + + while (!e || e->type != OBJ_TREE) + e = dereference(e, sha1); + return e; +} + +static void print_ls(int mode, const unsigned char *sha1, const char *path) +{ + static struct strbuf line = STRBUF_INIT; + + /* See show_tree(). */ + const char *type = + S_ISGITLINK(mode) ? commit_type : + S_ISDIR(mode) ? tree_type : + blob_type; + + if (!mode) { + /* missing SP path LF */ + strbuf_reset(&line); + strbuf_addstr(&line, "missing "); + quote_c_style(path, &line, NULL, 0); + strbuf_addch(&line, '\n'); + } else { + /* mode SP type SP object_name TAB path LF */ + strbuf_reset(&line); + strbuf_addf(&line, "%06o %s %s\t", + mode & ~NO_DELTA, type, sha1_to_hex(sha1)); + quote_c_style(path, &line, NULL, 0); + strbuf_addch(&line, '\n'); + } + cat_blob_write(line.buf, line.len); +} + +static void parse_ls(struct branch *b) +{ + const char *p; + struct tree_entry *root = NULL; + struct tree_entry leaf = {NULL}; + + /* ls SP ( SP)? */ + p = command_buf.buf + strlen("ls "); + if (*p == '"') { + if (!b) + die("Not in a commit: %s", command_buf.buf); + root = &b->branch_tree; + } else { + struct object_entry *e = parse_treeish_dataref(&p); + root = new_tree_entry(); + hashcpy(root->versions[1].sha1, e->idx.sha1); + load_tree(root); + if (*p++ != ' ') + die("Missing space after tree-ish: %s", command_buf.buf); + } + if (*p == '"') { + static struct strbuf uq = STRBUF_INIT; + const char *endp; + strbuf_reset(&uq); + if (unquote_c_style(&uq, p, &endp)) + die("Invalid path: %s", command_buf.buf); + if (*endp) + die("Garbage after path in: %s", command_buf.buf); + p = uq.buf; + } + tree_content_get(root, p, &leaf); + /* + * A directory in preparation would have a sha1 of zero + * until it is saved. Save, for simplicity. + */ + if (S_ISDIR(leaf.versions[1].mode)) + store_tree(&leaf); + + print_ls(leaf.versions[1].mode, leaf.versions[1].sha1, p); + if (!b || root != &b->branch_tree) + release_tree_entry(root); +} + static void checkpoint(void) { checkpoint_requested = 0; @@ -2854,7 +3054,8 @@ static char* make_fast_import_path(const char *path) return strbuf_detach(&abs_path, NULL); } -static void option_import_marks(const char *marks, int from_stream) +static void option_import_marks(const char *marks, + int from_stream, int ignore_missing) { if (import_marks_file) { if (from_stream) @@ -2868,6 +3069,7 @@ static void option_import_marks(const char *marks, int from_stream) import_marks_file = make_fast_import_path(marks); safe_create_leading_directories_const(import_marks_file); import_marks_file_from_stream = from_stream; + import_marks_file_ignore_missing = ignore_missing; } static void option_date_format(const char *fmt) @@ -2967,18 +3169,23 @@ static int parse_one_feature(const char *feature, int from_stream) if (!prefixcmp(feature, "date-format=")) { option_date_format(feature + 12); } else if (!prefixcmp(feature, "import-marks=")) { - option_import_marks(feature + 13, from_stream); + option_import_marks(feature + 13, from_stream, 0); + } else if (!prefixcmp(feature, "import-marks-if-exists=")) { + option_import_marks(feature + strlen("import-marks-if-exists="), + from_stream, 1); } else if (!prefixcmp(feature, "export-marks=")) { option_export_marks(feature + 13); } else if (!strcmp(feature, "cat-blob")) { ; /* Don't die - this feature is supported */ - } else if (!prefixcmp(feature, "relative-marks")) { + } else if (!strcmp(feature, "relative-marks")) { relative_marks_paths = 1; - } else if (!prefixcmp(feature, "no-relative-marks")) { + } else if (!strcmp(feature, "no-relative-marks")) { relative_marks_paths = 0; - } else if (!prefixcmp(feature, "force")) { + } else if (!strcmp(feature, "done")) { + require_explicit_termination = 1; + } else if (!strcmp(feature, "force")) { force_update = 1; - } else if (!strcmp(feature, "notes")) { + } else if (!strcmp(feature, "notes") || !strcmp(feature, "ls")) { ; /* do nothing; we have the feature */ } else { return 0; @@ -3032,20 +3239,16 @@ static int git_pack_config(const char *k, const char *v, void *cb) return 0; } if (!strcmp(k, "pack.indexversion")) { - pack_idx_default_version = git_config_int(k, v); - if (pack_idx_default_version > 2) + pack_idx_opts.version = git_config_int(k, v); + if (pack_idx_opts.version > 2) die("bad pack.indexversion=%"PRIu32, - pack_idx_default_version); + pack_idx_opts.version); return 0; } if (!strcmp(k, "pack.packsizelimit")) { max_packsize = git_config_ulong(k, v); return 0; } - if (!strcmp(k, "core.bigfilethreshold")) { - long n = git_config_int(k, v); - big_file_threshold = 0 < n ? n : 0; - } return git_default_config(k, v, cb); } @@ -3093,6 +3296,7 @@ int main(int argc, const char **argv) usage(fast_import_usage); setup_git_directory(); + reset_pack_idx_option(&pack_idx_opts); git_config(git_pack_config, NULL); if (!pack_compression_seen && core_compression_seen) pack_compression_level = core_compression_level; @@ -3119,6 +3323,8 @@ int main(int argc, const char **argv) while (read_next_command() != EOF) { if (!strcmp("blob", command_buf.buf)) parse_new_blob(); + else if (!prefixcmp(command_buf.buf, "ls ")) + parse_ls(NULL); else if (!prefixcmp(command_buf.buf, "commit ")) parse_new_commit(); else if (!prefixcmp(command_buf.buf, "tag ")) @@ -3127,6 +3333,8 @@ int main(int argc, const char **argv) parse_reset_branch(); else if (!strcmp("checkpoint", command_buf.buf)) parse_checkpoint(); + else if (!strcmp("done", command_buf.buf)) + break; else if (!prefixcmp(command_buf.buf, "progress ")) parse_progress(); else if (!prefixcmp(command_buf.buf, "feature ")) @@ -3146,6 +3354,9 @@ int main(int argc, const char **argv) if (!seen_data_command) parse_argv(); + if (require_explicit_termination && feof(stdin)) + die("stream ends early"); + end_packfile(); dump_branches(); @@ -3167,10 +3378,10 @@ int main(int argc, const char **argv) fprintf(stderr, "---------------------------------------------------------------------\n"); fprintf(stderr, "Alloc'd objects: %10" PRIuMAX "\n", alloc_count); fprintf(stderr, "Total objects: %10" PRIuMAX " (%10" PRIuMAX " duplicates )\n", total_count, duplicate_count); - fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB]); - fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE]); - fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT]); - fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG]); + fprintf(stderr, " blobs : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB], delta_count_by_type[OBJ_BLOB], delta_count_attempts_by_type[OBJ_BLOB]); + fprintf(stderr, " trees : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE], delta_count_by_type[OBJ_TREE], delta_count_attempts_by_type[OBJ_TREE]); + fprintf(stderr, " commits: %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT], delta_count_by_type[OBJ_COMMIT], delta_count_attempts_by_type[OBJ_COMMIT]); + fprintf(stderr, " tags : %10" PRIuMAX " (%10" PRIuMAX " duplicates %10" PRIuMAX " deltas of %10" PRIuMAX" attempts)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG], delta_count_by_type[OBJ_TAG], delta_count_attempts_by_type[OBJ_TAG]); fprintf(stderr, "Total branches: %10lu (%10lu loads )\n", branch_count, branch_load_count); fprintf(stderr, " marks: %10" PRIuMAX " (%10" PRIuMAX " unique )\n", (((uintmax_t)1) << marks->shift) * 1024, marks_set_count); fprintf(stderr, " atoms: %10u\n", atom_cnt);