Merge branch 'sp/maint-fast-import-large-blob' into sp/fast-import-large-blob

* sp/maint-fast-import-large-blob:
  fast-import: Stream very large blobs directly to pack
  bash: don't offer remote transport helpers as subcommands

Conflicts:
	fast-import.c

diff --cc Documentation/config.txt
Simple merge

diff --cc Documentation/git-fast-import.txt
Simple merge

diff --cc contrib/completion/git-completion.bash
Simple merge

diff --cc fast-import.c
index 901784fe911567752c872e5433f0295768654b5a,2105310c5412ac5afea99840080406340e26552b..ca210822dbe88ce82d6eaf9d435dd68561e90c9f
--- 1/fast-import.c
--- 2/fast-import.c
+++ b/fast-import.c
@@@ -2595,140 -2552,39 +2727,142 @@@ static void parse_progress(void
        skip_optional_lf();
  }
  
 -static void import_marks(const char *input_file)
 +static char* make_fast_import_path(const char *path)
  {
 -      char line[512];
 -      FILE *f = fopen(input_file, "r");
 -      if (!f)
 -              die_errno("cannot read '%s'", input_file);
 -      while (fgets(line, sizeof(line), f)) {
 -              uintmax_t mark;
 -              char *end;
 -              unsigned char sha1[20];
 -              struct object_entry *e;
 +      struct strbuf abs_path = STRBUF_INIT;
  
 -              end = strchr(line, '\n');
 -              if (line[0] != ':' || !end)
 -                      die("corrupt mark line: %s", line);
 -              *end = 0;
 -              mark = strtoumax(line + 1, &end, 10);
 -              if (!mark || end == line + 1
 -                      || *end != ' ' || get_sha1(end + 1, sha1))
 -                      die("corrupt mark line: %s", line);
 -              e = find_object(sha1);
 -              if (!e) {
 -                      enum object_type type = sha1_object_info(sha1, NULL);
 -                      if (type < 0)
 -                              die("object not found: %s", sha1_to_hex(sha1));
 -                      e = insert_object(sha1);
 -                      e->type = type;
 -                      e->pack_id = MAX_PACK_ID;
 -                      e->offset = 1; /* just not zero! */
 -              }
 -              insert_mark(mark, e);
 +      if (!relative_marks_paths || is_absolute_path(path))
 +              return xstrdup(path);
 +      strbuf_addf(&abs_path, "%s/info/fast-import/%s", get_git_dir(), path);
 +      return strbuf_detach(&abs_path, NULL);
 +}
 +
 +static void option_import_marks(const char *marks, int from_stream)
 +{
 +      if (import_marks_file) {
 +              if (from_stream)
 +                      die("Only one import-marks command allowed per stream");
 +
 +              /* read previous mark file */
 +              if(!import_marks_file_from_stream)
 +                      read_marks();
        }
 -      fclose(f);
 +
 +      import_marks_file = make_fast_import_path(marks);
 +      import_marks_file_from_stream = from_stream;
 +}
 +
 +static void option_date_format(const char *fmt)
 +{
 +      if (!strcmp(fmt, "raw"))
 +              whenspec = WHENSPEC_RAW;
 +      else if (!strcmp(fmt, "rfc2822"))
 +              whenspec = WHENSPEC_RFC2822;
 +      else if (!strcmp(fmt, "now"))
 +              whenspec = WHENSPEC_NOW;
 +      else
 +              die("unknown --date-format argument %s", fmt);
 +}
 +
 +static void option_max_pack_size(const char *packsize)
 +{
 +      max_packsize = strtoumax(packsize, NULL, 0) * 1024 * 1024;
 +}
 +
 +static void option_depth(const char *depth)
 +{
 +      max_depth = strtoul(depth, NULL, 0);
 +      if (max_depth > MAX_DEPTH)
 +              die("--depth cannot exceed %u", MAX_DEPTH);
 +}
 +
 +static void option_active_branches(const char *branches)
 +{
 +      max_active_branches = strtoul(branches, NULL, 0);
 +}
 +
 +static void option_export_marks(const char *marks)
 +{
 +      export_marks_file = make_fast_import_path(marks);
 +}
 +
 +static void option_export_pack_edges(const char *edges)
 +{
 +      if (pack_edges)
 +              fclose(pack_edges);
 +      pack_edges = fopen(edges, "a");
 +      if (!pack_edges)
 +              die_errno("Cannot open '%s'", edges);
 +}
 +
 +static int parse_one_option(const char *option)
 +{
 +      if (!prefixcmp(option, "max-pack-size=")) {
 +              option_max_pack_size(option + 14);
++      } else if (!prefixcmp(option, "big-file-threshold=")) {
++              big_file_threshold = strtoumax(option + 19, NULL, 0) * 1024 * 1024;
 +      } else if (!prefixcmp(option, "depth=")) {
 +              option_depth(option + 6);
 +      } else if (!prefixcmp(option, "active-branches=")) {
 +              option_active_branches(option + 16);
 +      } else if (!prefixcmp(option, "export-pack-edges=")) {
 +              option_export_pack_edges(option + 18);
 +      } else if (!prefixcmp(option, "quiet")) {
 +              show_stats = 0;
 +      } else if (!prefixcmp(option, "stats")) {
 +              show_stats = 1;
 +      } else {
 +              return 0;
 +      }
 +
 +      return 1;
 +}
 +
 +static int parse_one_feature(const char *feature, int from_stream)
 +{
 +      if (!prefixcmp(feature, "date-format=")) {
 +              option_date_format(feature + 12);
 +      } else if (!prefixcmp(feature, "import-marks=")) {
 +              option_import_marks(feature + 13, from_stream);
 +      } else if (!prefixcmp(feature, "export-marks=")) {
 +              option_export_marks(feature + 13);
 +      } else if (!prefixcmp(feature, "relative-marks")) {
 +              relative_marks_paths = 1;
 +      } else if (!prefixcmp(feature, "no-relative-marks")) {
 +              relative_marks_paths = 0;
 +      } else if (!prefixcmp(feature, "force")) {
 +              force_update = 1;
 +      } else {
 +              return 0;
 +      }
 +
 +      return 1;
 +}
 +
 +static void parse_feature(void)
 +{
 +      char *feature = command_buf.buf + 8;
 +
 +      if (seen_data_command)
 +              die("Got feature command '%s' after data command", feature);
 +
 +      if (parse_one_feature(feature, 1))
 +              return;
 +
 +      die("This version of fast-import does not support feature %s.", feature);
 +}
 +
 +static void parse_option(void)
 +{
 +      char *option = command_buf.buf + 11;
 +
 +      if (seen_data_command)
 +              die("Got option command '%s' after data command", option);
 +
 +      if (parse_one_option(option))
 +              return;
 +
 +      die("This version of fast-import does not support option: %s", option);
  }
  
  static int git_pack_config(const char *k, const char *v, void *cb)
@@@ -2753,37 -2613,11 +2891,37 @@@
  }
  
  static const char fast_import_usage[] =
- "git fast-import [--date-format=f] [--max-pack-size=n] [--depth=n] [--active-branches=n] [--export-marks=marks.file]";
+ "git fast-import [--date-format=f] [--max-pack-size=n] [--big-file-threshold=n] [--depth=n] [--active-branches=n] [--export-marks=marks.file]";
  
 +static void parse_argv(void)
 +{
 +      unsigned int i;
 +
 +      for (i = 1; i < global_argc; i++) {
 +              const char *a = global_argv[i];
 +
 +              if (*a != '-' || !strcmp(a, "--"))
 +                      break;
 +
 +              if (parse_one_option(a + 2))
 +                      continue;
 +
 +              if (parse_one_feature(a + 2, 0))
 +                      continue;
 +
 +              die("unknown option %s", a);
 +      }
 +      if (i != global_argc)
 +              usage(fast_import_usage);
 +
 +      seen_data_command = 1;
 +      if (import_marks_file)
 +              read_marks();
 +}
 +
  int main(int argc, const char **argv)
  {
 -      unsigned int i, show_stats = 1;
 +      unsigned int i;
  
        git_extract_argv0_path(argv[0]);
  

diff --cc t/t9300-fast-import.sh
index 60d6f5d1ba7d5421c4c364e070e35bdb024b3b4a,513db86ad28227b5264b16a659802c5c59bbd0ea..131f03298809ad193cc75ab77deda6daaf713d1f
--- 1/t/t9300-fast-import.sh
--- 2/t/t9300-fast-import.sh
+++ b/t/t9300-fast-import.sh
@@@ -1292,248 -1251,53 +1292,294 @@@ test_expect_success 
  
  echo "$note3_data" >expect
  test_expect_success \
 -      'Q: verify note for third commit' \
 -      'git cat-file blob refs/notes/foobar:$commit3 >actual && test_cmp expect actual'
 +      'Q: verify first note for third commit' \
 +      'git cat-file blob refs/notes/foobar~2:$commit3 >actual && test_cmp expect actual'
 +
 +cat >expect <<EOF
 +parent `git rev-parse --verify refs/notes/foobar~2`
 +author $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +
 +notes (:10)
 +EOF
 +test_expect_success \
 +      'Q: verify second notes commit' \
 +      'git cat-file commit refs/notes/foobar^ | sed 1d >actual &&
 +      test_cmp expect actual'
 +
 +cat >expect.unsorted <<EOF
 +100644 blob $commit1
 +100644 blob $commit2
 +100644 blob $commit3
 +EOF
 +cat expect.unsorted | sort >expect
 +test_expect_success \
 +      'Q: verify second notes tree' \
 +      'git cat-file -p refs/notes/foobar^^{tree} | sed "s/ [0-9a-f]*  / /" >actual &&
 +       test_cmp expect actual'
 +
 +echo "$note1b_data" >expect
 +test_expect_success \
 +      'Q: verify second note for first commit' \
 +      'git cat-file blob refs/notes/foobar^:$commit1 >actual && test_cmp expect actual'
 +
 +echo "$note2_data" >expect
 +test_expect_success \
 +      'Q: verify first note for second commit' \
 +      'git cat-file blob refs/notes/foobar^:$commit2 >actual && test_cmp expect actual'
 +
 +echo "$note3_data" >expect
 +test_expect_success \
 +      'Q: verify first note for third commit' \
 +      'git cat-file blob refs/notes/foobar^:$commit3 >actual && test_cmp expect actual'
 +
 +cat >expect <<EOF
 +author $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +
 +notes (:11)
 +EOF
 +test_expect_success \
 +      'Q: verify third notes commit' \
 +      'git cat-file commit refs/notes/foobar2 | sed 1d >actual &&
 +      test_cmp expect actual'
 +
 +cat >expect.unsorted <<EOF
 +100644 blob $commit1
 +EOF
 +cat expect.unsorted | sort >expect
 +test_expect_success \
 +      'Q: verify third notes tree' \
 +      'git cat-file -p refs/notes/foobar2^{tree} | sed "s/ [0-9a-f]*  / /" >actual &&
 +       test_cmp expect actual'
 +
 +echo "$note1c_data" >expect
 +test_expect_success \
 +      'Q: verify third note for first commit' \
 +      'git cat-file blob refs/notes/foobar2:$commit1 >actual && test_cmp expect actual'
 +
 +cat >expect <<EOF
 +parent `git rev-parse --verify refs/notes/foobar^`
 +author $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
 +
 +notes (:12)
 +EOF
 +test_expect_success \
 +      'Q: verify fourth notes commit' \
 +      'git cat-file commit refs/notes/foobar | sed 1d >actual &&
 +      test_cmp expect actual'
 +
 +cat >expect.unsorted <<EOF
 +100644 blob $commit2
 +EOF
 +cat expect.unsorted | sort >expect
 +test_expect_success \
 +      'Q: verify fourth notes tree' \
 +      'git cat-file -p refs/notes/foobar^{tree} | sed "s/ [0-9a-f]*   / /" >actual &&
 +       test_cmp expect actual'
 +
 +echo "$note2b_data" >expect
 +test_expect_success \
 +      'Q: verify second note for second commit' \
 +      'git cat-file blob refs/notes/foobar:$commit2 >actual && test_cmp expect actual'
 +
 +###
 +### series R (feature and option)
 +###
 +
 +cat >input <<EOF
 +feature no-such-feature-exists
 +EOF
 +
 +test_expect_success 'R: abort on unsupported feature' '
 +      test_must_fail git fast-import <input
 +'
 +
 +cat >input <<EOF
 +feature date-format=now
 +EOF
 +
 +test_expect_success 'R: supported feature is accepted' '
 +      git fast-import <input
 +'
 +
 +cat >input << EOF
 +blob
 +data 3
 +hi
 +feature date-format=now
 +EOF
 +
 +test_expect_success 'R: abort on receiving feature after data command' '
 +      test_must_fail git fast-import <input
 +'
 +
 +cat >input << EOF
 +feature import-marks=git.marks
 +feature import-marks=git2.marks
 +EOF
 +
 +test_expect_success 'R: only one import-marks feature allowed per stream' '
 +      test_must_fail git fast-import <input
 +'
 +
 +cat >input << EOF
 +feature export-marks=git.marks
 +blob
 +mark :1
 +data 3
 +hi
 +
 +EOF
 +
 +test_expect_success \
 +    'R: export-marks feature results in a marks file being created' \
 +    'cat input | git fast-import &&
 +    grep :1 git.marks'
 +
 +test_expect_success \
 +    'R: export-marks options can be overriden by commandline options' \
 +    'cat input | git fast-import --export-marks=other.marks &&
 +    grep :1 other.marks'
 +
 +cat >input << EOF
 +feature import-marks=marks.out
 +feature export-marks=marks.new
 +EOF
 +
 +test_expect_success \
 +    'R: import to output marks works without any content' \
 +    'cat input | git fast-import &&
 +    test_cmp marks.out marks.new'
 +
 +cat >input <<EOF
 +feature import-marks=nonexistant.marks
 +feature export-marks=marks.new
 +EOF
 +
 +test_expect_success \
 +    'R: import marks prefers commandline marks file over the stream' \
 +    'cat input | git fast-import --import-marks=marks.out &&
 +    test_cmp marks.out marks.new'
 +
 +
 +cat >input <<EOF
 +feature import-marks=nonexistant.marks
 +feature export-marks=combined.marks
 +EOF
 +
 +test_expect_success 'R: multiple --import-marks= should be honoured' '
 +    head -n2 marks.out > one.marks &&
 +    tail -n +3 marks.out > two.marks &&
 +    git fast-import --import-marks=one.marks --import-marks=two.marks <input &&
 +    test_cmp marks.out combined.marks
 +'
 +
 +cat >input <<EOF
 +feature relative-marks
 +feature import-marks=relative.in
 +feature export-marks=relative.out
 +EOF
 +
 +test_expect_success 'R: feature relative-marks should be honoured' '
 +    mkdir -p .git/info/fast-import/ &&
 +    cp marks.new .git/info/fast-import/relative.in &&
 +    git fast-import <input &&
 +    test_cmp marks.new .git/info/fast-import/relative.out
 +'
 +
 +cat >input <<EOF
 +feature relative-marks
 +feature import-marks=relative.in
 +feature no-relative-marks
 +feature export-marks=non-relative.out
 +EOF
 +
 +test_expect_success 'R: feature no-relative-marks should be honoured' '
 +    git fast-import <input &&
 +    test_cmp marks.new non-relative.out
 +'
 +
 +cat >input << EOF
 +option git quiet
 +blob
 +data 3
 +hi
 +
 +EOF
 +
 +touch empty
 +
 +test_expect_success 'R: quiet option results in no stats being output' '
 +    cat input | git fast-import 2> output &&
 +    test_cmp empty output
 +'
 +
 +cat >input <<EOF
 +option git non-existing-option
 +EOF
 +
 +test_expect_success 'R: die on unknown option' '
 +    test_must_fail git fast-import <input
 +'
 +
 +test_expect_success 'R: unknown commandline options are rejected' '\
 +    test_must_fail git fast-import --non-existing-option < /dev/null
 +'
 +
 +cat >input <<EOF
 +option non-existing-vcs non-existing-option
 +EOF
 +
 +test_expect_success 'R: ignore non-git options' '
 +    git fast-import <input
 +'
  
+ ##
+ ## R: very large blobs
+ ##
+ blobsize=$((2*1024*1024 + 53))
+ test-genrandom bar $blobsize >expect
+ cat >input <<INPUT_END
+ commit refs/heads/big-file
+ committer $GIT_COMMITTER_NAME <$GIT_COMMITTER_EMAIL> $GIT_COMMITTER_DATE
+ data <<COMMIT
+ R - big file
+ COMMIT
+ 
+ M 644 inline big1
+ data $blobsize
+ INPUT_END
+ cat expect >>input
+ cat >>input <<INPUT_END
+ M 644 inline big2
+ data $blobsize
+ INPUT_END
+ cat expect >>input
+ echo >>input
+ 
+ test_expect_success \
+       'R: blob bigger than threshold' \
+       'test_create_repo R &&
+        git --git-dir=R/.git fast-import --big-file-threshold=1 <input'
+ test_expect_success \
+       'R: verify created pack' \
+       ': >verify &&
+        for p in R/.git/objects/pack/*.pack;
+        do
+          git verify-pack -v $p >>verify || exit;
+        done'
+ test_expect_success \
+       'R: verify written objects' \
+       'git --git-dir=R/.git cat-file blob big-file:big1 >actual &&
+        test_cmp expect actual &&
+        a=$(git --git-dir=R/.git rev-parse big-file:big1) &&
+        b=$(git --git-dir=R/.git rev-parse big-file:big2) &&
+        test $a = $b'
+ test_expect_success \
+       'R: blob appears only once' \
+       'n=$(grep $a verify | wc -l) &&
+        test 1 = $n'
+ 
  test_done