git-svnimport.perl

   1 #!/usr/bin/perl -w
   2
   3 # This tool is copyright (c) 2005, Matthias Urlichs.
   4 # It is released under the Gnu Public License, version 2.
   5 #
   6 # The basic idea is to pull and analyze SVN changes.
   7 #
   8 # Checking out the files is done by a single long-running CVS connection
   9 # / server process.
  10 #
  11 # The head revision is on branch "origin" by default.
  12 # You can change that with the '-o' option.
  13
  14 require v5.8.0; # for shell-safe open("-|",LIST)
  15 use strict;
  16 use warnings;
  17 use Getopt::Std;
  18 use File::Spec;
  19 use File::Temp qw(tempfile);
  20 use File::Path qw(mkpath);
  21 use File::Basename qw(basename dirname);
  22 use Time::Local;
  23 use IO::Pipe;
  24 use POSIX qw(strftime dup2);
  25 use IPC::Open2;
  26 use SVN::Core;
  27 use SVN::Ra;
  28
  29 $SIG{'PIPE'}="IGNORE";
  30 $ENV{'TZ'}="UTC";
  31
  32 our($opt_h,$opt_o,$opt_v,$opt_u,$opt_C,$opt_i,$opt_m,$opt_M,$opt_t,$opt_T,$opt_b);
  33
  34 sub usage() {
  35         print STDERR <<END;
  36 Usage: ${\basename $0}     # fetch/update GIT from CVS
  37        [-o branch-for-HEAD] [-h] [-v]
  38        [-C GIT_repository] [-t tagname] [-T trunkname] [-b branchname]
  39        [-i] [-u] [-s subst] [-m] [-M regex] [SVN_URL]
  40 END
  41         exit(1);
  42 }
  43
  44 getopts("b:C:hivmM:o:t:T:u") or usage();
  45 usage if $opt_h;
  46
  47 my $tag_name = $opt_t || "tags";
  48 my $trunk_name = $opt_T || "trunk";
  49 my $branch_name = $opt_b || "branches";
  50
  51 @ARGV <= 1 or usage();
  52
  53 $opt_o ||= "origin";
  54 my $git_tree = $opt_C;
  55 $git_tree ||= ".";
  56
  57 my $cvs_tree;
  58 if ($#ARGV == 0) {
  59         $cvs_tree = $ARGV[0];
  60 } elsif (-f 'CVS/Repository') {
  61         open my $f, '<', 'CVS/Repository' or
  62             die 'Failed to open CVS/Repository';
  63         $cvs_tree = <$f>;
  64         chomp $cvs_tree;
  65         close $f;
  66 } else {
  67         usage();
  68 }
  69
  70 our @mergerx = ();
  71 if ($opt_m) {
  72         @mergerx = ( qr/\W(?:from|of|merge|merging|merged) (\w+)/i );
  73 }
  74 if ($opt_M) {
  75         push (@mergerx, qr/$opt_M/);
  76 }
  77
  78 select(STDERR); $|=1; select(STDOUT);
  79
  80
  81 package SVNconn;
  82 # Basic SVN connection.
  83 # We're only interested in connecting and downloading, so ...
  84
  85 use File::Spec;
  86 use File::Temp qw(tempfile);
  87 use POSIX qw(strftime dup2);
  88
  89 sub new {
  90         my($what,$repo) = @_;
  91         $what=ref($what) if ref($what);
  92
  93         my $self = {};
  94         $self->{'buffer'} = "";
  95         bless($self,$what);
  96
  97         $repo =~ s#/+$##;
  98         $self->{'fullrep'} = $repo;
  99         $self->conn();
 100
 101         $self->{'lines'} = undef;
 102
 103         return $self;
 104 }
 105
 106 sub conn {
 107         my $self = shift;
 108         my $repo = $self->{'fullrep'};
 109         my $s = SVN::Ra->new($repo);
 110
 111         die "SVN connection to $repo: $!\n" unless defined $s;
 112         $self->{'svn'} = $s;
 113         $self->{'repo'} = $repo;
 114         $self->{'maxrev'} = $s->get_latest_revnum();
 115 }
 116
 117 sub file {
 118         my($self,$path,$rev) = @_;
 119         my $res;
 120
 121         my ($fh, $name) = tempfile('gitsvn.XXXXXX',
 122                     DIR => File::Spec->tmpdir(), UNLINK => 1);
 123
 124         print "... $rev $path ...\n" if $opt_v;
 125         eval { $self->{'svn'}->get_file($path,$rev,$fh); };
 126         if (defined $@ and $@ !~ /Attempted to get checksum/) {
 127             # retry
 128             $self->conn();
 129                 eval { $self->{'svn'}->get_file($path,$rev,$fh); };
 130         };
 131         return () if defined $@ and $@ !~ /Attempted to get checksum/;
 132         die $@ if $@;
 133         close ($fh);
 134
 135         return ($name, $res);
 136 }
 137
 138
 139 package main;
 140
 141 my $svn = SVNconn->new($cvs_tree);
 142
 143
 144 sub pdate($) {
 145         my($d) = @_;
 146         $d =~ m#(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)#
 147                 or die "Unparseable date: $d\n";
 148         my $y=$1; $y-=1900 if $y>1900;
 149         return timegm($6||0,$5,$4,$3,$2-1,$y);
 150 }
 151
 152 sub getwd() {
 153         my $pwd = `pwd`;
 154         chomp $pwd;
 155         return $pwd;
 156 }
 157
 158
 159 sub get_headref($$) {
 160     my $name    = shift;
 161     my $git_dir = shift;
 162     my $sha;
 163
 164     if (open(C,"$git_dir/refs/heads/$name")) {
 165         chomp($sha = <C>);
 166         close(C);
 167         length($sha) == 40
 168             or die "Cannot get head id for $name ($sha): $!\n";
 169     }
 170     return $sha;
 171 }
 172
 173
 174 -d $git_tree
 175         or mkdir($git_tree,0777)
 176         or die "Could not create $git_tree: $!";
 177 chdir($git_tree);
 178
 179 my $orig_branch = "";
 180 my $forward_master = 0;
 181 my %branches;
 182
 183 my $git_dir = $ENV{"GIT_DIR"} || ".git";
 184 $git_dir = getwd()."/".$git_dir unless $git_dir =~ m#^/#;
 185 $ENV{"GIT_DIR"} = $git_dir;
 186 my $orig_git_index;
 187 $orig_git_index = $ENV{GIT_INDEX_FILE} if exists $ENV{GIT_INDEX_FILE};
 188 my ($git_ih, $git_index) = tempfile('gitXXXXXX', SUFFIX => '.idx',
 189                                     DIR => File::Spec->tmpdir());
 190 close ($git_ih);
 191 $ENV{GIT_INDEX_FILE} = $git_index;
 192 my $maxnum = 0;
 193 my $last_rev = "";
 194 my $last_branch;
 195 my $current_rev = 0;
 196 unless(-d $git_dir) {
 197         system("git-init-db");
 198         die "Cannot init the GIT db at $git_tree: $?\n" if $?;
 199         system("git-read-tree");
 200         die "Cannot init an empty tree: $?\n" if $?;
 201
 202         $last_branch = $opt_o;
 203         $orig_branch = "";
 204 } else {
 205         -f "$git_dir/refs/heads/$opt_o"
 206                 or die "Branch '$opt_o' does not exist.\n".
 207                        "Either use the correct '-o branch' option,\n".
 208                        "or import to a new repository.\n";
 209
 210         -f "$git_dir/svn2git"
 211                 or die "'$git_dir/svn2git' does not exist.\n".
 212                        "You need that file for incremental imports.\n";
 213         $last_branch = basename(readlink("$git_dir/HEAD"));
 214         unless($last_branch) {
 215                 warn "Cannot read the last branch name: $! -- assuming 'master'\n";
 216                 $last_branch = "master";
 217         }
 218         $orig_branch = $last_branch;
 219         $last_rev = get_headref($orig_branch, $git_dir);
 220         if (-f "$git_dir/SVN2GIT_HEAD") {
 221                 die <<EOM;
 222 SVN2GIT_HEAD exists.
 223 Make sure your working directory corresponds to HEAD and remove SVN2GIT_HEAD.
 224 You may need to run
 225
 226     git-read-tree -m -u SVN2GIT_HEAD HEAD
 227 EOM
 228         }
 229         system('cp', "$git_dir/HEAD", "$git_dir/SVN2GIT_HEAD");
 230
 231         $forward_master =
 232             $opt_o ne 'master' && -f "$git_dir/refs/heads/master" &&
 233             system('cmp', '-s', "$git_dir/refs/heads/master",
 234                                 "$git_dir/refs/heads/$opt_o") == 0;
 235
 236         # populate index
 237         system('git-read-tree', $last_rev);
 238         die "read-tree failed: $?\n" if $?;
 239
 240         # Get the last import timestamps
 241         open my $B,"<", "$git_dir/svn2git";
 242         while(<$B>) {
 243                 chomp;
 244                 my($num,$branch,$ref) = split;
 245                 $branches{$branch}{$num} = $ref;
 246                 $branches{$branch}{"LAST"} = $ref;
 247                 $current_rev = $num+1 if $current_rev < $num+1;
 248         }
 249         close($B);
 250 }
 251 -d $git_dir
 252         or die "Could not create git subdir ($git_dir).\n";
 253
 254 open BRANCHES,">>", "$git_dir/svn2git";
 255
 256
 257 ## cvsps output:
 258 #---------------------
 259 #PatchSet 314
 260 #Date: 1999/09/18 13:03:59
 261 #Author: wkoch
 262 #Branch: STABLE-BRANCH-1-0
 263 #Ancestor branch: HEAD
 264 #Tag: (none)
 265 #Log:
 266 #    See ChangeLog: Sat Sep 18 13:03:28 CEST 1999  Werner Koch
 267 #Members:
 268 #       README:1.57->1.57.2.1
 269 #       VERSION:1.96->1.96.2.1
 270 #
 271 #---------------------
 272
 273 my $state = 0;
 274
 275 sub get_file($$$) {
 276         my($rev,$branch,$path) = @_;
 277
 278         # revert split_path(), below
 279         my $svnpath;
 280         $path = "" if $path eq "/"; # this should not happen, but ...
 281         if($branch eq "/") {
 282                 $svnpath = "/$trunk_name/$path";
 283         } elsif($branch =~ m#^/#) {
 284                 $svnpath = "/$tag_name$branch/$path";
 285         } else {
 286                 $svnpath = "/$branch_name/$branch/$path";
 287         }
 288
 289         # now get it
 290         my ($name, $res) = eval { $svn->file($svnpath,$rev); };
 291         return () unless defined $name;
 292
 293         open my $F, '-|', "git-hash-object", "-w", $name
 294                 or die "Cannot create object: $!\n";
 295         my $sha = <$F>;
 296         chomp $sha;
 297         close $F;
 298         my $mode = "0644"; # SV does not seem to store any file modes
 299         return [$mode, $sha, $path];
 300 }
 301
 302 sub split_path($$) {
 303         my($rev,$path) = @_;
 304         my $branch;
 305
 306         if($path =~ s#^/\Q$tag_name\E/([^/]+)/?##) {
 307                 $branch = "/$1";
 308         } elsif($path =~ s#^/\Q$trunk_name\E/?##) {
 309                 $branch = "/";
 310         } elsif($path =~ s#^/\Q$branch_name\E/([^/]+)/?##) {
 311                 $branch = $1;
 312         } else {
 313                 print STDERR "$rev: Unrecognized path: $path\n";
 314                 return ()
 315         }
 316         $path = "/" if $path eq "";
 317         return ($branch,$path);
 318 }
 319
 320 sub commit {
 321         my($branch, $changed_paths, $revision, $author, $date, $message) = @_;
 322         my($author_name,$author_email,$dest);
 323         my(@old,@new);
 324
 325         if (not defined $author) {
 326                 $author_name = $author_email = "unknown";
 327         } elsif ($author =~ /^(.*?)\s+<(.*)>$/) {
 328                 ($author_name, $author_email) = ($1, $2);
 329         } else {
 330                 $author =~ s/^<(.*)>$/$1/;
 331                 $author_name = $author_email = $author;
 332         }
 333         $date = pdate($date);
 334
 335         my $tag;
 336         my $parent;
 337         if($branch eq "/") { # trunk
 338                 $parent = $opt_o;
 339         } elsif($branch =~ m#^/(.+)#) { # tag
 340                 $tag = 1;
 341                 $parent = $1;
 342         } else { # "normal" branch
 343                 # nothing to do
 344                 $parent = $branch;
 345         }
 346         $dest = $parent;
 347
 348         my $prev = $changed_paths->{"/"};
 349         if($prev and $prev->[0] eq "A") {
 350                 delete $changed_paths->{"/"};
 351                 my $oldpath = $prev->[1];
 352                 my $rev;
 353                 if(defined $oldpath) {
 354                         my $p;
 355                         ($parent,$p) = split_path($revision,$oldpath);
 356                         if($parent eq "/") {
 357                                 $parent = $opt_o;
 358                         } else {
 359                                 $parent =~ s#^/##; # if it's a tag
 360                         }
 361                 } else {
 362                         $parent = undef;
 363                 }
 364         }
 365
 366         my $rev;
 367         if(defined $parent) {
 368                 open(H,"git-rev-parse --verify $parent |");
 369                 $rev = <H>;
 370                 close(H) or do {
 371                         print STDERR "$revision: cannot find commit '$parent'!\n";
 372                         return;
 373                 };
 374                 chop $rev;
 375                 if(length($rev) != 40) {
 376                         print STDERR "$revision: cannot find commit '$parent'!\n";
 377                         return;
 378                 }
 379                 $rev = $branches{($parent eq $opt_o) ? "/" : $parent}{"LAST"};
 380                 if($revision != 1 and not $rev) {
 381                         print STDERR "$revision: do not know ancestor for '$parent'!\n";
 382                         return;
 383                 }
 384         } else {
 385                 $rev = undef;
 386         }
 387
 388 #       if($prev and $prev->[0] eq "A") {
 389 #               if(not $tag) {
 390 #                       unless(open(H,"> $git_dir/refs/heads/$branch")) {
 391 #                               print STDERR "$revision: Could not create branch $branch: $!\n";
 392 #                               $state=11;
 393 #                               next;
 394 #                       }
 395 #                       print H "$rev\n"
 396 #                               or die "Could not write branch $branch: $!";
 397 #                       close(H)
 398 #                               or die "Could not write branch $branch: $!";
 399 #               }
 400 #       }
 401         if(not defined $rev) {
 402                 unlink($git_index);
 403         } elsif ($rev ne $last_rev) {
 404                 print "Switching from $last_rev to $rev ($branch)\n" if $opt_v;
 405                 system("git-read-tree", $rev);
 406                 die "read-tree failed for $rev: $?\n" if $?;
 407                 $last_rev = $rev;
 408         }
 409
 410         while(my($path,$action) = each %$changed_paths) {
 411                 if ($action->[0] eq "A") {
 412                         my $f = get_file($revision,$branch,$path);
 413                         push(@new,$f) if $f;
 414                 } elsif ($action->[0] eq "D") {
 415                         push(@old,$path);
 416                 } elsif ($action->[0] eq "M") {
 417                         my $f = get_file($revision,$branch,$path);
 418                         push(@new,$f) if $f;
 419                 } elsif ($action->[0] eq "R") {
 420                         # refer to a file/tree in an earlier commit
 421                         push(@old,$path); # remove any old stuff
 422
 423                         # ... and add any new stuff
 424                         my($b,$p) = split_path($revision,$action->[1]);
 425                         open my $F,"-|","git-ls-tree","-r","-z", $branches{$b}{$action->[2]}, $p;
 426                         local $/ = '\0';
 427                         while(<$F>) {
 428                                 chomp;
 429                                 my($m,$p) = split(/\t/,$_,2);
 430                                 my($mode,$type,$sha1) = split(/ /,$m);
 431                                 next if $type ne "blob";
 432                                 push(@new,[$mode,$sha1,$p]);
 433                         }
 434                 } else {
 435                         die "$revision: unknown action '".$action->[0]."' for $path\n";
 436                 }
 437         }
 438
 439         if(@old) {
 440                 open my $F, "-│", "git-ls-files", "-z", @old or die $!;
 441                 @old = ();
 442                 local $/ = '\0';
 443                 while(<$F>) {
 444                         chomp;
 445                         push(@old,$_);
 446                 }
 447                 close($F);
 448
 449                 while(@old) {
 450                         my @o2;
 451                         if(@old > 55) {
 452                                 @o2 = splice(@old,0,50);
 453                         } else {
 454                                 @o2 = @old;
 455                                 @old = ();
 456                         }
 457                         system("git-update-index","--force-remove","--",@o2);
 458                         die "Cannot remove files: $?\n" if $?;
 459                 }
 460         }
 461         while(@new) {
 462                 my @n2;
 463                 if(@new > 12) {
 464                         @n2 = splice(@new,0,10);
 465                 } else {
 466                         @n2 = @new;
 467                         @new = ();
 468                 }
 469                 system("git-update-index","--add",
 470                         (map { ('--cacheinfo', @$_) } @n2));
 471                 die "Cannot add files: $?\n" if $?;
 472         }
 473
 474         my $pid = open(C,"-|");
 475         die "Cannot fork: $!" unless defined $pid;
 476         unless($pid) {
 477                 exec("git-write-tree");
 478                 die "Cannot exec git-write-tree: $!\n";
 479         }
 480         chomp(my $tree = <C>);
 481         length($tree) == 40
 482                 or die "Cannot get tree id ($tree): $!\n";
 483         close(C)
 484                 or die "Error running git-write-tree: $?\n";
 485         print "Tree ID $tree\n" if $opt_v;
 486
 487         my $pr = IO::Pipe->new() or die "Cannot open pipe: $!\n";
 488         my $pw = IO::Pipe->new() or die "Cannot open pipe: $!\n";
 489         $pid = fork();
 490         die "Fork: $!\n" unless defined $pid;
 491         unless($pid) {
 492                 $pr->writer();
 493                 $pw->reader();
 494                 open(OUT,">&STDOUT");
 495                 dup2($pw->fileno(),0);
 496                 dup2($pr->fileno(),1);
 497                 $pr->close();
 498                 $pw->close();
 499
 500                 my @par = ();
 501                 @par = ("-p",$rev) if defined $rev;
 502
 503                 # loose detection of merges
 504                 # based on the commit msg
 505                 foreach my $rx (@mergerx) {
 506                         if ($message =~ $rx) {
 507                                 my $mparent = $1;
 508                                 if ($mparent eq 'HEAD') { $mparent = $opt_o };
 509                                 if ( -e "$git_dir/refs/heads/$mparent") {
 510                                         $mparent = get_headref($mparent, $git_dir);
 511                                         push @par, '-p', $mparent;
 512                                         print OUT "Merge parent branch: $mparent\n" if $opt_v;
 513                                 }
 514                         }
 515                 }
 516
 517                 exec("env",
 518                         "GIT_AUTHOR_NAME=$author_name",
 519                         "GIT_AUTHOR_EMAIL=$author_email",
 520                         "GIT_AUTHOR_DATE=".strftime("+0000 %Y-%m-%d %H:%M:%S",gmtime($date)),
 521                         "GIT_COMMITTER_NAME=$author_name",
 522                         "GIT_COMMITTER_EMAIL=$author_email",
 523                         "GIT_COMMITTER_DATE=".strftime("+0000 %Y-%m-%d %H:%M:%S",gmtime($date)),
 524                         "git-commit-tree", $tree,@par);
 525                 die "Cannot exec git-commit-tree: $!\n";
 526         }
 527         $pw->writer();
 528         $pr->reader();
 529
 530         $message =~ s/[\s\n]+\z//;
 531
 532         print $pw "$message\n"
 533                 or die "Error writing to git-commit-tree: $!\n";
 534         $pw->close();
 535
 536         print "Committed change $revision:$branch ".strftime("%Y-%m-%d %H:%M:%S",gmtime($date)).")\n" if $opt_v;
 537         chomp(my $cid = <$pr>);
 538         length($cid) == 40
 539                 or die "Cannot get commit id ($cid): $!\n";
 540         print "Commit ID $cid\n" if $opt_v;
 541         $pr->close();
 542
 543         waitpid($pid,0);
 544         die "Error running git-commit-tree: $?\n" if $?;
 545
 546         if(defined $dest) {
 547                 print "Writing to refs/heads/$dest\n" if $opt_v;
 548                 open(C,">$git_dir/refs/heads/$dest") and
 549                 print C ("$cid\n") and
 550                 close(C)
 551                         or die "Cannot write branch $dest for update: $!\n";
 552         } else {
 553                 print "... no known parent\n" if $opt_v;
 554         }
 555         $branches{$branch}{"LAST"} = $cid;
 556         $branches{$branch}{$revision} = $cid;
 557         $last_rev = $cid;
 558         print BRANCHES "$revision $branch $cid\n";
 559         print "DONE: $revision $dest $cid\n" if $opt_v;
 560
 561         if($tag) {
 562                 my($in, $out) = ('','');
 563                 $last_rev = "-" if %$changed_paths;
 564                 # the tag was 'complex', i.e. did not refer to a "real" revision
 565
 566                 $tag =~ tr/_/\./ if $opt_u;
 567
 568                 my $pid = open2($in, $out, 'git-mktag');
 569                 print $out ("object $cid\n".
 570                     "type commit\n".
 571                     "tag $tag\n".
 572                     "tagger $author_name <$author_email>\n") and
 573                 close($out)
 574                     or die "Cannot create tag object $tag: $!\n";
 575
 576                 my $tagobj = <$in>;
 577                 chomp $tagobj;
 578
 579                 if ( !close($in) or waitpid($pid, 0) != $pid or
 580                                 $? != 0 or $tagobj !~ /^[0123456789abcdef]{40}$/ ) {
 581                         die "Cannot create tag object $tag: $!\n";
 582                 }
 583
 584
 585                 open(C,">$git_dir/refs/tags/$tag")
 586                         or die "Cannot create tag $tag: $!\n";
 587                 print C "$tagobj\n"
 588                         or die "Cannot write tag $tag: $!\n";
 589                 close(C)
 590                         or die "Cannot write tag $tag: $!\n";
 591
 592                 print "Created tag '$tag' on '$branch'\n" if $opt_v;
 593         }
 594 }
 595
 596 my ($changed_paths, $revision, $author, $date, $message, $pool) = @_;
 597 sub _commit_all {
 598         ($changed_paths, $revision, $author, $date, $message, $pool) = @_;
 599         my %p;
 600         while(my($path,$action) = each %$changed_paths) {
 601                 $p{$path} = [ $action->action,$action->copyfrom_path, $action->copyfrom_rev ];
 602         }
 603         $changed_paths = \%p;
 604 }
 605
 606 sub commit_all {
 607         my %done;
 608         my @col;
 609         my $pref;
 610         my $branch;
 611
 612         while(my($path,$action) = each %$changed_paths) {
 613                 ($branch,$path) = split_path($revision,$path);
 614                 next if not defined $branch;
 615                 $done{$branch}{$path} = $action;
 616         }
 617         while(($branch,$changed_paths) = each %done) {
 618                 commit($branch, $changed_paths, $revision, $author, $date, $message);
 619         }
 620 }
 621
 622 while(++$current_rev < $svn->{'maxrev'}) {
 623         $svn->{'svn'}->get_log("/",$current_rev,$current_rev,$current_rev,1,1,\&_commit_all,"");
 624         commit_all();
 625 }
 626
 627
 628 unlink($git_index);
 629
 630 if (defined $orig_git_index) {
 631         $ENV{GIT_INDEX_FILE} = $orig_git_index;
 632 } else {
 633         delete $ENV{GIT_INDEX_FILE};
 634 }
 635
 636 # Now switch back to the branch we were in before all of this happened
 637 if($orig_branch) {
 638         print "DONE\n" if $opt_v;
 639         system("cp","$git_dir/refs/heads/$opt_o","$git_dir/refs/heads/master")
 640                 if $forward_master;
 641         unless ($opt_i) {
 642                 system('git-read-tree', '-m', '-u', 'SVN2GIT_HEAD', 'HEAD');
 643                 die "read-tree failed: $?\n" if $?;
 644         }
 645 } else {
 646         $orig_branch = "master";
 647         print "DONE; creating $orig_branch branch\n" if $opt_v;
 648         system("cp","$git_dir/refs/heads/$opt_o","$git_dir/refs/heads/master")
 649                 unless -f "$git_dir/refs/heads/master";
 650         unlink("$git_dir/HEAD");
 651         symlink("refs/heads/$orig_branch","$git_dir/HEAD");
 652         unless ($opt_i) {
 653                 system('git checkout');
 654                 die "checkout failed: $?\n" if $?;
 655         }
 656 }
 657 unlink("$git_dir/SVN2GIT_HEAD");
 658 close(BRANCHES);