git-svn: add --follow-parent and --no-metadata options to fetch

--follow-parent:
  This is especially helpful when we're tracking a directory
  that has been moved around within the repository, or if we
  started tracking a branch and never tracked the trunk it was
  descended from.

  This relies on the SVN::* libraries to work.  We can't
  reliably parse path info from the svn command-line client
  without relying on XML, so it's better just to have the SVN::*
  libs installed.

  This also removes oldvalue verification when calling update-ref

  In SVN, branches can be deleted, and then recreated under the
  same path as the original one with different ancestry
  information, causing parent information to be mismatched /
  misordered.

  Also force the current ref, if existing, to be a parent,
  regardless of whether or not it was specified.

--no-metadata:
  This gets rid of the git-svn-id: lines at the end of every commit.

  With this, you lose the ability to use the rebuild command.  If
  you ever lose your .git/svn/git-svn/.rev_db file, you won't be
  able to fetch again, either.  This is fine for one-shot imports.

  Also fix some issues with multi-fetch --follow-parent that were
  exposed while testing this.  Additionally, repack checking is
  simplified greatly.

  git-svn log will not work on repositories using this, either.

Signed-off-by: Eric Wong <normalperson@yhbt.net>
Signed-off-by: Junio C Hamano <junkio@cox.net>

diff --git a/contrib/git-svn/git-svn.perl b/contrib/git-svn/git-svn.perl
index 4bdc9766cdd04cb8a62f437c71fa04b4738312d7..a865a11a3eba6798980d55b061cee593a2f405b6 100755 (executable)
--- a/contrib/git-svn/git-svn.perl
+++ b/contrib/git-svn/git-svn.perl
@@ -19,6 +19,7 @@ my $TZ = $ENV{TZ};
 # make sure the svn binary gives consistent output between locales and TZs:
 $ENV{TZ} = 'UTC';
 $ENV{LC_ALL} = 'C';
+$| = 1; # unbuffer STDOUT
 
 # If SVN:: library support is added, please make the dependencies
 # optional and preserve the capability to use the command-line client.
@@ -46,7 +47,7 @@ my $sha1_short = qr/[a-f\d]{4,40}/;
 my ($_revision,$_stdin,$_no_ignore_ext,$_no_stop_copy,$_help,$_rmdir,$_edit,
        $_find_copies_harder, $_l, $_cp_similarity, $_cp_remote,
        $_repack, $_repack_nr, $_repack_flags,
-       $_message, $_file,
+       $_message, $_file, $_follow_parent, $_no_metadata,
        $_template, $_shared, $_no_default_regex, $_no_graft_copy,
        $_limit, $_verbose, $_incremental, $_oneline, $_l_fmt, $_show_commit,
        $_version, $_upgrade, $_authors, $_branch_all_refs, @_opt_m);
@@ -56,9 +57,11 @@ my @repo_path_split_cache;
 
 my %fc_opts = ( 'no-ignore-externals' => \$_no_ignore_ext,
                'branch|b=s' => \@_branch_from,
+               'follow-parent|follow' => \$_follow_parent,
                'branch-all-refs|B' => \$_branch_all_refs,
                'authors-file|A=s' => \$_authors,
                'repack:i' => \$_repack,
+               'no-metadata' => \$_no_metadata,
                'repack-flags|repack-args|repack-opts=s' => \$_repack_flags);
 
 my ($_trunk, $_tags, $_branches);
@@ -824,35 +827,19 @@ sub fetch_child_id {
        my $id = shift;
        print "Fetching $id\n";
        my $ref = "$GIT_DIR/refs/remotes/$id";
-       my $ca = file_to_s($ref) if (-r $ref);
-       defined(my $pid = fork) or croak $!;
+       defined(my $pid = open my $fh, '-|') or croak $!;
        if (!$pid) {
+               $_repack = undef;
                $GIT_SVN = $ENV{GIT_SVN_ID} = $id;
                init_vars();
                fetch(@_);
                exit 0;
        }
-       waitpid $pid, 0;
-       croak $? if $?;
-       return unless $_repack || -r $ref;
-
-       my $cb = file_to_s($ref);
-
-       defined($pid = open my $fh, '-|') or croak $!;
-       my $url = file_to_s("$GIT_DIR/svn/$id/info/url");
-       $url = qr/\Q$url\E/;
-       if (!$pid) {
-               exec qw/git-rev-list --pretty=raw/,
-                               $ca ? "$ca..$cb" : $cb or croak $!;
-       }
        while (<$fh>) {
-               if (/^    git-svn-id: $url\@\d+ [a-f0-9\-]+$/) {
-                       check_repack();
-               } elsif (/^    git-svn-id: \S+\@\d+ [a-f0-9\-]+$/) {
-                       last;
-               }
+               print $_;
+               check_repack() if (/^r\d+ = $sha1/);
        }
-       close $fh;
+       close $fh or croak $?;
 }
 
 sub rec_fetch {
@@ -1919,6 +1906,13 @@ sub git_commit {
                croak $? if $?;
                restore_index($index);
        }
+
+       # just in case we clobber the existing ref, we still want that ref
+       # as our parent:
+       if (my $cur = eval { file_to_s("$GIT_DIR/refs/remotes/$GIT_SVN") }) {
+               push @tmp_parents, $cur;
+       }
+
        if (exists $tree_map{$tree}) {
                foreach my $p (@{$tree_map{$tree}}) {
                        my $skip;
@@ -1949,31 +1943,26 @@ sub git_commit {
                last if @exec_parents > 16;
        }
 
-       defined(my $pid = open my $out_fh, '-|') or croak $!;
-       if ($pid == 0) {
-               my $msg_fh = IO::File->new_tmpfile or croak $!;
-               print $msg_fh $log_msg->{msg}, "\ngit-svn-id: ",
-                                       "$SVN_URL\@$log_msg->{revision}",
+       set_commit_env($log_msg);
+       my @exec = ('git-commit-tree', $tree);
+       push @exec, '-p', $_  foreach @exec_parents;
+       defined(my $pid = open3(my $msg_fh, my $out_fh, '>&STDERR', @exec))
+                                                               or croak $!;
+       print $msg_fh $log_msg->{msg} or croak $!;
+       unless ($_no_metadata) {
+               print $msg_fh "\ngit-svn-id: $SVN_URL\@$log_msg->{revision}",
                                        " $SVN_UUID\n" or croak $!;
-               $msg_fh->flush == 0 or croak $!;
-               seek $msg_fh, 0, 0 or croak $!;
-               set_commit_env($log_msg);
-               my @exec = ('git-commit-tree',$tree);
-               push @exec, '-p', $_  foreach @exec_parents;
-               open STDIN, '<&', $msg_fh or croak $!;
-               exec @exec or croak $!;
        }
+       $msg_fh->flush == 0 or croak $!;
+       close $msg_fh or croak $!;
        chomp(my $commit = do { local $/; <$out_fh> });
-       close $out_fh or croak $?;
+       close $out_fh or croak $!;
+       waitpid $pid, 0;
+       croak $? if $?;
        if ($commit !~ /^$sha1$/o) {
-               croak "Failed to commit, invalid sha1: $commit\n";
+               die "Failed to commit, invalid sha1: $commit\n";
        }
-       my @update_ref = ('git-update-ref',"refs/remotes/$GIT_SVN",$commit);
-       if (my $primary_parent = shift @exec_parents) {
-               quiet_run(qw/git-rev-parse --verify/,"refs/remotes/$GIT_SVN^0");
-               push @update_ref, $primary_parent unless $?;
-       }
-       sys(@update_ref);
+       sys('git-update-ref',"refs/remotes/$GIT_SVN",$commit);
        revdb_set($REVDB, $log_msg->{revision}, $commit);
 
        # this output is read via pipe, do not change:
@@ -2058,6 +2047,11 @@ sub safe_qx {
 }
 
 sub svn_compat_check {
+       if ($_follow_parent) {
+               print STDERR 'E: --follow-parent functionality is only ',
+                               "available when SVN libraries are used\n";
+               exit 1;
+       }
        my @co_help = safe_qx(qw(svn co -h));
        unless (grep /ignore-externals/,@co_help) {
                print STDERR "W: Installed svn version does not support ",
@@ -2386,6 +2380,28 @@ sub write_grafts {
        close $fh or croak $!;
 }
 
+sub read_url_paths_all {
+       my ($l_map, $pfx, $p) = @_;
+       my @dir;
+       foreach (<$p/*>) {
+               if (-r "$_/info/url") {
+                       $pfx .= '/' if $pfx && $pfx !~ m!/$!;
+                       my $id = $pfx . basename $_;
+                       my $url = file_to_s("$_/info/url");
+                       my ($u, $p) = repo_path_split($url);
+                       $l_map->{$u}->{$p} = $id;
+               } elsif (-d $_) {
+                       push @dir, $_;
+               }
+       }
+       foreach (@dir) {
+               my $x = $_;
+               $x =~ s!^\Q$GIT_DIR\E/svn/!!o;
+               read_url_paths_all($l_map, $x, $_);
+       }
+}
+
+# this one only gets ids that have been imported, not new ones
 sub read_url_paths {
        my $l_map = {};
        git_svn_each(sub { my $x = shift;
@@ -2599,7 +2615,6 @@ sub libsvn_get_file {
        # redirect STDOUT for SVN 1.1.x compatibility
        open my $stdout, '>&', \*STDOUT or croak $!;
        open STDOUT, '>&', $in or croak $!;
-       $| = 1; # not sure if this is necessary, better safe than sorry...
        my ($r, $props) = $SVN->get_file($f, $rev, \*STDOUT, $pool);
        $in->flush == 0 or croak $!;
        open STDOUT, '>&', $stdout or croak $!;
@@ -2702,6 +2717,28 @@ sub svn_grab_base_rev {
        close $fh;
        if (defined $c && length $c) {
                my ($url, $rev, $uuid) = cmt_metadata($c);
+               return ($rev, $c) if defined $rev;
+       }
+       if ($_no_metadata) {
+               my $offset = -41; # from tail
+               my $rl;
+               open my $fh, '<', $REVDB or
+                       die "--no-metadata specified and $REVDB not readable\n";
+               seek $fh, $offset, 2;
+               $rl = readline $fh;
+               defined $rl or return (undef, undef);
+               chomp $rl;
+               while ($c ne $rl && tell $fh != 0) {
+                       $offset -= 41;
+                       seek $fh, $offset, 2;
+                       $rl = readline $fh;
+                       defined $rl or return (undef, undef);
+                       chomp $rl;
+               }
+               my $rev = tell $fh;
+               croak $! if ($rev < -1);
+               $rev =  ($rev - 41) / 41;
+               close $fh or croak $!;
                return ($rev, $c);
        }
        return (undef, undef);
@@ -2799,15 +2836,45 @@ sub libsvn_find_parent_branch {
        print STDERR  "Found possible branch point: ",
                                "$branch_from => $svn_path, $r\n";
        $branch_from =~ s#^/##;
-       my $l_map = read_url_paths();
+       my $l_map = {};
+       read_url_paths_all($l_map, '', "$GIT_DIR/svn");
        my $url = $SVN->{url};
        defined $l_map->{$url} or return;
-       my $id = $l_map->{$url}->{$branch_from} or return;
+       my $id = $l_map->{$url}->{$branch_from};
+       if (!defined $id && $_follow_parent) {
+               print STDERR "Following parent: $branch_from\@$r\n";
+               # auto create a new branch and follow it
+               $id = basename($branch_from);
+               $id .= '@'.$r if -r "$GIT_DIR/svn/$id";
+               while (-r "$GIT_DIR/svn/$id") {
+                       # just grow a tail if we're not unique enough :x
+                       $id .= '-';
+               }
+       }
+       return unless defined $id;
+
        my ($r0, $parent) = find_rev_before($r,$id,1);
+       if ($_follow_parent && (!defined $r0 || !defined $parent)) {
+               defined(my $pid = fork) or croak $!;
+               if (!$pid) {
+                       $GIT_SVN = $ENV{GIT_SVN_ID} = $id;
+                       init_vars();
+                       $SVN_URL = "$url/$branch_from";
+                       $SVN_LOG = $SVN = undef;
+                       setup_git_svn();
+                       # we can't assume SVN_URL exists at r+1:
+                       $_revision = "0:$r";
+                       fetch_lib();
+                       exit 0;
+               }
+               waitpid $pid, 0;
+               croak $? if $?;
+               ($r0, $parent) = find_rev_before($r,$id,1);
+       }
        return unless (defined $r0 && defined $parent);
        if (revisions_eq($branch_from, $r0, $r)) {
                unlink $GIT_SVN_INDEX;
-               print STDERR "Found branch parent: $parent\n";
+               print STDERR "Found branch parent: ($GIT_SVN) $parent\n";
                sys(qw/git-read-tree/, $parent);
                return libsvn_fetch($parent, $paths, $rev,
                                        $author, $date, $msg);
@@ -3274,6 +3341,16 @@ diff-index line ($m hash)
 }
 ;
 
+# retval of read_url_paths{,_all}();
+$l_map = {
+       # repository root url
+       'https://svn.musicpd.org' => {
+               # repository path               # GIT_SVN_ID
+               'mpd/trunk'             =>      'trunk',
+               'mpd/tags/0.11.5'       =>      'tags/0.11.5',
+       },
+}
+
 Notes:
        I don't trust the each() function on unless I created %hash myself
        because the internal iterator may not have started at base.

diff --git a/contrib/git-svn/t/t0004-follow-parent.sh b/contrib/git-svn/t/t0004-follow-parent.sh
new file mode 100644 (file)
index 0000000..01488ff
--- /dev/null
+++ b/contrib/git-svn/t/t0004-follow-parent.sh
@@ -0,0 +1,44 @@
+#!/bin/sh
+#
+# Copyright (c) 2006 Eric Wong
+#
+
+test_description='git-svn --follow-parent fetching'
+. ./lib-git-svn.sh
+
+if test -n "$GIT_SVN_NO_LIB" && test "$GIT_SVN_NO_LIB" -ne 0
+then
+       echo 'Skipping: --follow-parent needs SVN libraries'
+       test_done
+       exit 0
+fi
+
+test_expect_success 'initialize repo' "
+       mkdir import &&
+       cd import &&
+       mkdir -p trunk &&
+       echo hello > trunk/readme &&
+       svn import -m 'initial' . $svnrepo &&
+       cd .. &&
+       svn co $svnrepo wc &&
+       cd wc &&
+       echo world >> trunk/readme &&
+       svn commit -m 'another commit' &&
+       svn up &&
+       svn mv -m 'rename to thunk' trunk thunk &&
+       svn up &&
+       echo goodbye >> thunk/readme &&
+       svn commit -m 'bye now' &&
+       cd ..
+       "
+
+test_expect_success 'init and fetch --follow-parent a moved directory' "
+       git-svn init -i thunk $svnrepo/thunk &&
+       git-svn fetch --follow-parent -i thunk &&
+       git-rev-parse --verify refs/remotes/trunk &&
+       test '$?' -eq '0'
+       "
+
+test_debug 'gitk --all &'
+
+test_done