contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = ["p4"]
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += ["-u",user]
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += ["-P", password]
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += ["-p", port]
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += ["-h", host]
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += ["-c", client]
  46
  47
  48     if isinstance(cmd,basestring):
  49         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50     else:
  51         real_cmd += cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56     # not using the shell, we have to set it ourselves.  This path could
  57     # be relative, so go there first, then figure out where we ended up.
  58     os.chdir(dir)
  59     os.environ['PWD'] = os.getcwd()
  60
  61 def die(msg):
  62     if verbose:
  63         raise Exception(msg)
  64     else:
  65         sys.stderr.write(msg + "\n")
  66         sys.exit(1)
  67
  68 def write_pipe(c, stdin):
  69     if verbose:
  70         sys.stderr.write('Writing pipe: %s\n' % str(c))
  71
  72     expand = isinstance(c,basestring)
  73     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  74     pipe = p.stdin
  75     val = pipe.write(stdin)
  76     pipe.close()
  77     if p.wait():
  78         die('Command failed: %s' % str(c))
  79
  80     return val
  81
  82 def p4_write_pipe(c, stdin):
  83     real_cmd = p4_build_cmd(c)
  84     return write_pipe(real_cmd, stdin)
  85
  86 def read_pipe(c, ignore_error=False):
  87     if verbose:
  88         sys.stderr.write('Reading pipe: %s\n' % str(c))
  89
  90     expand = isinstance(c,basestring)
  91     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  92     pipe = p.stdout
  93     val = pipe.read()
  94     if p.wait() and not ignore_error:
  95         die('Command failed: %s' % str(c))
  96
  97     return val
  98
  99 def p4_read_pipe(c, ignore_error=False):
 100     real_cmd = p4_build_cmd(c)
 101     return read_pipe(real_cmd, ignore_error)
 102
 103 def read_pipe_lines(c):
 104     if verbose:
 105         sys.stderr.write('Reading pipe: %s\n' % str(c))
 106
 107     expand = isinstance(c, basestring)
 108     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 109     pipe = p.stdout
 110     val = pipe.readlines()
 111     if pipe.close() or p.wait():
 112         die('Command failed: %s' % str(c))
 113
 114     return val
 115
 116 def p4_read_pipe_lines(c):
 117     """Specifically invoke p4 on the command supplied. """
 118     real_cmd = p4_build_cmd(c)
 119     return read_pipe_lines(real_cmd)
 120
 121 def system(cmd):
 122     expand = isinstance(cmd,basestring)
 123     if verbose:
 124         sys.stderr.write("executing %s\n" % str(cmd))
 125     subprocess.check_call(cmd, shell=expand)
 126
 127 def p4_system(cmd):
 128     """Specifically invoke p4 as the system command. """
 129     real_cmd = p4_build_cmd(cmd)
 130     expand = isinstance(real_cmd, basestring)
 131     subprocess.check_call(real_cmd, shell=expand)
 132
 133 def p4_integrate(src, dest):
 134     p4_system(["integrate", "-Dt", src, dest])
 135
 136 def p4_sync(path):
 137     p4_system(["sync", path])
 138
 139 def p4_add(f):
 140     p4_system(["add", f])
 141
 142 def p4_delete(f):
 143     p4_system(["delete", f])
 144
 145 def p4_edit(f):
 146     p4_system(["edit", f])
 147
 148 def p4_revert(f):
 149     p4_system(["revert", f])
 150
 151 def p4_reopen(type, file):
 152     p4_system(["reopen", "-t", type, file])
 153
 154 #
 155 # Canonicalize the p4 type and return a tuple of the
 156 # base type, plus any modifiers.  See "p4 help filetypes"
 157 # for a list and explanation.
 158 #
 159 def split_p4_type(p4type):
 160
 161     p4_filetypes_historical = {
 162         "ctempobj": "binary+Sw",
 163         "ctext": "text+C",
 164         "cxtext": "text+Cx",
 165         "ktext": "text+k",
 166         "kxtext": "text+kx",
 167         "ltext": "text+F",
 168         "tempobj": "binary+FSw",
 169         "ubinary": "binary+F",
 170         "uresource": "resource+F",
 171         "uxbinary": "binary+Fx",
 172         "xbinary": "binary+x",
 173         "xltext": "text+Fx",
 174         "xtempobj": "binary+Swx",
 175         "xtext": "text+x",
 176         "xunicode": "unicode+x",
 177         "xutf16": "utf16+x",
 178     }
 179     if p4type in p4_filetypes_historical:
 180         p4type = p4_filetypes_historical[p4type]
 181     mods = ""
 182     s = p4type.split("+")
 183     base = s[0]
 184     mods = ""
 185     if len(s) > 1:
 186         mods = s[1]
 187     return (base, mods)
 188
 189
 190 def setP4ExecBit(file, mode):
 191     # Reopens an already open file and changes the execute bit to match
 192     # the execute bit setting in the passed in mode.
 193
 194     p4Type = "+x"
 195
 196     if not isModeExec(mode):
 197         p4Type = getP4OpenedType(file)
 198         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 199         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 200         if p4Type[-1] == "+":
 201             p4Type = p4Type[0:-1]
 202
 203     p4_reopen(p4Type, file)
 204
 205 def getP4OpenedType(file):
 206     # Returns the perforce file type for the given file.
 207
 208     result = p4_read_pipe(["opened", file])
 209     match = re.match(".*\((.+)\)\r?$", result)
 210     if match:
 211         return match.group(1)
 212     else:
 213         die("Could not determine file type for %s (result: '%s')" % (file, result))
 214
 215 def diffTreePattern():
 216     # This is a simple generator for the diff tree regex pattern. This could be
 217     # a class variable if this and parseDiffTreeEntry were a part of a class.
 218     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 219     while True:
 220         yield pattern
 221
 222 def parseDiffTreeEntry(entry):
 223     """Parses a single diff tree entry into its component elements.
 224
 225     See git-diff-tree(1) manpage for details about the format of the diff
 226     output. This method returns a dictionary with the following elements:
 227
 228     src_mode - The mode of the source file
 229     dst_mode - The mode of the destination file
 230     src_sha1 - The sha1 for the source file
 231     dst_sha1 - The sha1 fr the destination file
 232     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 233     status_score - The score for the status (applicable for 'C' and 'R'
 234                    statuses). This is None if there is no score.
 235     src - The path for the source file.
 236     dst - The path for the destination file. This is only present for
 237           copy or renames. If it is not present, this is None.
 238
 239     If the pattern is not matched, None is returned."""
 240
 241     match = diffTreePattern().next().match(entry)
 242     if match:
 243         return {
 244             'src_mode': match.group(1),
 245             'dst_mode': match.group(2),
 246             'src_sha1': match.group(3),
 247             'dst_sha1': match.group(4),
 248             'status': match.group(5),
 249             'status_score': match.group(6),
 250             'src': match.group(7),
 251             'dst': match.group(10)
 252         }
 253     return None
 254
 255 def isModeExec(mode):
 256     # Returns True if the given git mode represents an executable file,
 257     # otherwise False.
 258     return mode[-3:] == "755"
 259
 260 def isModeExecChanged(src_mode, dst_mode):
 261     return isModeExec(src_mode) != isModeExec(dst_mode)
 262
 263 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 264
 265     if isinstance(cmd,basestring):
 266         cmd = "-G " + cmd
 267         expand = True
 268     else:
 269         cmd = ["-G"] + cmd
 270         expand = False
 271
 272     cmd = p4_build_cmd(cmd)
 273     if verbose:
 274         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 275
 276     # Use a temporary file to avoid deadlocks without
 277     # subprocess.communicate(), which would put another copy
 278     # of stdout into memory.
 279     stdin_file = None
 280     if stdin is not None:
 281         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 282         if isinstance(stdin,basestring):
 283             stdin_file.write(stdin)
 284         else:
 285             for i in stdin:
 286                 stdin_file.write(i + '\n')
 287         stdin_file.flush()
 288         stdin_file.seek(0)
 289
 290     p4 = subprocess.Popen(cmd,
 291                           shell=expand,
 292                           stdin=stdin_file,
 293                           stdout=subprocess.PIPE)
 294
 295     result = []
 296     try:
 297         while True:
 298             entry = marshal.load(p4.stdout)
 299             if cb is not None:
 300                 cb(entry)
 301             else:
 302                 result.append(entry)
 303     except EOFError:
 304         pass
 305     exitCode = p4.wait()
 306     if exitCode != 0:
 307         entry = {}
 308         entry["p4ExitCode"] = exitCode
 309         result.append(entry)
 310
 311     return result
 312
 313 def p4Cmd(cmd):
 314     list = p4CmdList(cmd)
 315     result = {}
 316     for entry in list:
 317         result.update(entry)
 318     return result;
 319
 320 def p4Where(depotPath):
 321     if not depotPath.endswith("/"):
 322         depotPath += "/"
 323     depotPath = depotPath + "..."
 324     outputList = p4CmdList(["where", depotPath])
 325     output = None
 326     for entry in outputList:
 327         if "depotFile" in entry:
 328             if entry["depotFile"] == depotPath:
 329                 output = entry
 330                 break
 331         elif "data" in entry:
 332             data = entry.get("data")
 333             space = data.find(" ")
 334             if data[:space] == depotPath:
 335                 output = entry
 336                 break
 337     if output == None:
 338         return ""
 339     if output["code"] == "error":
 340         return ""
 341     clientPath = ""
 342     if "path" in output:
 343         clientPath = output.get("path")
 344     elif "data" in output:
 345         data = output.get("data")
 346         lastSpace = data.rfind(" ")
 347         clientPath = data[lastSpace + 1:]
 348
 349     if clientPath.endswith("..."):
 350         clientPath = clientPath[:-3]
 351     return clientPath
 352
 353 def currentGitBranch():
 354     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 355
 356 def isValidGitDir(path):
 357     if (os.path.exists(path + "/HEAD")
 358         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 359         return True;
 360     return False
 361
 362 def parseRevision(ref):
 363     return read_pipe("git rev-parse %s" % ref).strip()
 364
 365 def extractLogMessageFromGitCommit(commit):
 366     logMessage = ""
 367
 368     ## fixme: title is first line of commit, not 1st paragraph.
 369     foundTitle = False
 370     for log in read_pipe_lines("git cat-file commit %s" % commit):
 371        if not foundTitle:
 372            if len(log) == 1:
 373                foundTitle = True
 374            continue
 375
 376        logMessage += log
 377     return logMessage
 378
 379 def extractSettingsGitLog(log):
 380     values = {}
 381     for line in log.split("\n"):
 382         line = line.strip()
 383         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 384         if not m:
 385             continue
 386
 387         assignments = m.group(1).split (':')
 388         for a in assignments:
 389             vals = a.split ('=')
 390             key = vals[0].strip()
 391             val = ('='.join (vals[1:])).strip()
 392             if val.endswith ('\"') and val.startswith('"'):
 393                 val = val[1:-1]
 394
 395             values[key] = val
 396
 397     paths = values.get("depot-paths")
 398     if not paths:
 399         paths = values.get("depot-path")
 400     if paths:
 401         values['depot-paths'] = paths.split(',')
 402     return values
 403
 404 def gitBranchExists(branch):
 405     proc = subprocess.Popen(["git", "rev-parse", branch],
 406                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 407     return proc.wait() == 0;
 408
 409 _gitConfig = {}
 410 def gitConfig(key, args = None): # set args to "--bool", for instance
 411     if not _gitConfig.has_key(key):
 412         argsFilter = ""
 413         if args != None:
 414             argsFilter = "%s " % args
 415         cmd = "git config %s%s" % (argsFilter, key)
 416         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 417     return _gitConfig[key]
 418
 419 def gitConfigList(key):
 420     if not _gitConfig.has_key(key):
 421         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 422     return _gitConfig[key]
 423
 424 def p4BranchesInGit(branchesAreInRemotes = True):
 425     branches = {}
 426
 427     cmdline = "git rev-parse --symbolic "
 428     if branchesAreInRemotes:
 429         cmdline += " --remotes"
 430     else:
 431         cmdline += " --branches"
 432
 433     for line in read_pipe_lines(cmdline):
 434         line = line.strip()
 435
 436         ## only import to p4/
 437         if not line.startswith('p4/') or line == "p4/HEAD":
 438             continue
 439         branch = line
 440
 441         # strip off p4
 442         branch = re.sub ("^p4/", "", line)
 443
 444         branches[branch] = parseRevision(line)
 445     return branches
 446
 447 def findUpstreamBranchPoint(head = "HEAD"):
 448     branches = p4BranchesInGit()
 449     # map from depot-path to branch name
 450     branchByDepotPath = {}
 451     for branch in branches.keys():
 452         tip = branches[branch]
 453         log = extractLogMessageFromGitCommit(tip)
 454         settings = extractSettingsGitLog(log)
 455         if settings.has_key("depot-paths"):
 456             paths = ",".join(settings["depot-paths"])
 457             branchByDepotPath[paths] = "remotes/p4/" + branch
 458
 459     settings = None
 460     parent = 0
 461     while parent < 65535:
 462         commit = head + "~%s" % parent
 463         log = extractLogMessageFromGitCommit(commit)
 464         settings = extractSettingsGitLog(log)
 465         if settings.has_key("depot-paths"):
 466             paths = ",".join(settings["depot-paths"])
 467             if branchByDepotPath.has_key(paths):
 468                 return [branchByDepotPath[paths], settings]
 469
 470         parent = parent + 1
 471
 472     return ["", settings]
 473
 474 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 475     if not silent:
 476         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 477                % localRefPrefix)
 478
 479     originPrefix = "origin/p4/"
 480
 481     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 482         line = line.strip()
 483         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 484             continue
 485
 486         headName = line[len(originPrefix):]
 487         remoteHead = localRefPrefix + headName
 488         originHead = line
 489
 490         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 491         if (not original.has_key('depot-paths')
 492             or not original.has_key('change')):
 493             continue
 494
 495         update = False
 496         if not gitBranchExists(remoteHead):
 497             if verbose:
 498                 print "creating %s" % remoteHead
 499             update = True
 500         else:
 501             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 502             if settings.has_key('change') > 0:
 503                 if settings['depot-paths'] == original['depot-paths']:
 504                     originP4Change = int(original['change'])
 505                     p4Change = int(settings['change'])
 506                     if originP4Change > p4Change:
 507                         print ("%s (%s) is newer than %s (%s). "
 508                                "Updating p4 branch from origin."
 509                                % (originHead, originP4Change,
 510                                   remoteHead, p4Change))
 511                         update = True
 512                 else:
 513                     print ("Ignoring: %s was imported from %s while "
 514                            "%s was imported from %s"
 515                            % (originHead, ','.join(original['depot-paths']),
 516                               remoteHead, ','.join(settings['depot-paths'])))
 517
 518         if update:
 519             system("git update-ref %s %s" % (remoteHead, originHead))
 520
 521 def originP4BranchesExist():
 522         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 523
 524 def p4ChangesForPaths(depotPaths, changeRange):
 525     assert depotPaths
 526     cmd = ['changes']
 527     for p in depotPaths:
 528         cmd += ["%s...%s" % (p, changeRange)]
 529     output = p4_read_pipe_lines(cmd)
 530
 531     changes = {}
 532     for line in output:
 533         changeNum = int(line.split(" ")[1])
 534         changes[changeNum] = True
 535
 536     changelist = changes.keys()
 537     changelist.sort()
 538     return changelist
 539
 540 def p4PathStartsWith(path, prefix):
 541     # This method tries to remedy a potential mixed-case issue:
 542     #
 543     # If UserA adds  //depot/DirA/file1
 544     # and UserB adds //depot/dira/file2
 545     #
 546     # we may or may not have a problem. If you have core.ignorecase=true,
 547     # we treat DirA and dira as the same directory
 548     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 549     if ignorecase:
 550         return path.lower().startswith(prefix.lower())
 551     return path.startswith(prefix)
 552
 553 class Command:
 554     def __init__(self):
 555         self.usage = "usage: %prog [options]"
 556         self.needsGit = True
 557
 558 class P4UserMap:
 559     def __init__(self):
 560         self.userMapFromPerforceServer = False
 561
 562     def getUserCacheFilename(self):
 563         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 564         return home + "/.gitp4-usercache.txt"
 565
 566     def getUserMapFromPerforceServer(self):
 567         if self.userMapFromPerforceServer:
 568             return
 569         self.users = {}
 570         self.emails = {}
 571
 572         for output in p4CmdList("users"):
 573             if not output.has_key("User"):
 574                 continue
 575             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 576             self.emails[output["Email"]] = output["User"]
 577
 578
 579         s = ''
 580         for (key, val) in self.users.items():
 581             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 582
 583         open(self.getUserCacheFilename(), "wb").write(s)
 584         self.userMapFromPerforceServer = True
 585
 586     def loadUserMapFromCache(self):
 587         self.users = {}
 588         self.userMapFromPerforceServer = False
 589         try:
 590             cache = open(self.getUserCacheFilename(), "rb")
 591             lines = cache.readlines()
 592             cache.close()
 593             for line in lines:
 594                 entry = line.strip().split("\t")
 595                 self.users[entry[0]] = entry[1]
 596         except IOError:
 597             self.getUserMapFromPerforceServer()
 598
 599 class P4Debug(Command):
 600     def __init__(self):
 601         Command.__init__(self)
 602         self.options = [
 603             optparse.make_option("--verbose", dest="verbose", action="store_true",
 604                                  default=False),
 605             ]
 606         self.description = "A tool to debug the output of p4 -G."
 607         self.needsGit = False
 608         self.verbose = False
 609
 610     def run(self, args):
 611         j = 0
 612         for output in p4CmdList(args):
 613             print 'Element: %d' % j
 614             j += 1
 615             print output
 616         return True
 617
 618 class P4RollBack(Command):
 619     def __init__(self):
 620         Command.__init__(self)
 621         self.options = [
 622             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 623             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 624         ]
 625         self.description = "A tool to debug the multi-branch import. Don't use :)"
 626         self.verbose = False
 627         self.rollbackLocalBranches = False
 628
 629     def run(self, args):
 630         if len(args) != 1:
 631             return False
 632         maxChange = int(args[0])
 633
 634         if "p4ExitCode" in p4Cmd("changes -m 1"):
 635             die("Problems executing p4");
 636
 637         if self.rollbackLocalBranches:
 638             refPrefix = "refs/heads/"
 639             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 640         else:
 641             refPrefix = "refs/remotes/"
 642             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 643
 644         for line in lines:
 645             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 646                 line = line.strip()
 647                 ref = refPrefix + line
 648                 log = extractLogMessageFromGitCommit(ref)
 649                 settings = extractSettingsGitLog(log)
 650
 651                 depotPaths = settings['depot-paths']
 652                 change = settings['change']
 653
 654                 changed = False
 655
 656                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 657                                                            for p in depotPaths]))) == 0:
 658                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 659                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 660                     continue
 661
 662                 while change and int(change) > maxChange:
 663                     changed = True
 664                     if self.verbose:
 665                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 666                     system("git update-ref %s \"%s^\"" % (ref, ref))
 667                     log = extractLogMessageFromGitCommit(ref)
 668                     settings =  extractSettingsGitLog(log)
 669
 670
 671                     depotPaths = settings['depot-paths']
 672                     change = settings['change']
 673
 674                 if changed:
 675                     print "%s rewound to %s" % (ref, change)
 676
 677         return True
 678
 679 class P4Submit(Command, P4UserMap):
 680     def __init__(self):
 681         Command.__init__(self)
 682         P4UserMap.__init__(self)
 683         self.options = [
 684                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 685                 optparse.make_option("--origin", dest="origin"),
 686                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 687                 # preserve the user, requires relevant p4 permissions
 688                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 689         ]
 690         self.description = "Submit changes from git to the perforce depot."
 691         self.usage += " [name of git branch to submit into perforce depot]"
 692         self.interactive = True
 693         self.origin = ""
 694         self.detectRenames = False
 695         self.verbose = False
 696         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 697         self.isWindows = (platform.system() == "Windows")
 698         self.myP4UserId = None
 699
 700     def check(self):
 701         if len(p4CmdList("opened ...")) > 0:
 702             die("You have files opened with perforce! Close them before starting the sync.")
 703
 704     # replaces everything between 'Description:' and the next P4 submit template field with the
 705     # commit message
 706     def prepareLogMessage(self, template, message):
 707         result = ""
 708
 709         inDescriptionSection = False
 710
 711         for line in template.split("\n"):
 712             if line.startswith("#"):
 713                 result += line + "\n"
 714                 continue
 715
 716             if inDescriptionSection:
 717                 if line.startswith("Files:") or line.startswith("Jobs:"):
 718                     inDescriptionSection = False
 719                 else:
 720                     continue
 721             else:
 722                 if line.startswith("Description:"):
 723                     inDescriptionSection = True
 724                     line += "\n"
 725                     for messageLine in message.split("\n"):
 726                         line += "\t" + messageLine + "\n"
 727
 728             result += line + "\n"
 729
 730         return result
 731
 732     def p4UserForCommit(self,id):
 733         # Return the tuple (perforce user,git email) for a given git commit id
 734         self.getUserMapFromPerforceServer()
 735         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 736         gitEmail = gitEmail.strip()
 737         if not self.emails.has_key(gitEmail):
 738             return (None,gitEmail)
 739         else:
 740             return (self.emails[gitEmail],gitEmail)
 741
 742     def checkValidP4Users(self,commits):
 743         # check if any git authors cannot be mapped to p4 users
 744         for id in commits:
 745             (user,email) = self.p4UserForCommit(id)
 746             if not user:
 747                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 748                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 749                     print "%s" % msg
 750                 else:
 751                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 752
 753     def lastP4Changelist(self):
 754         # Get back the last changelist number submitted in this client spec. This
 755         # then gets used to patch up the username in the change. If the same
 756         # client spec is being used by multiple processes then this might go
 757         # wrong.
 758         results = p4CmdList("client -o")        # find the current client
 759         client = None
 760         for r in results:
 761             if r.has_key('Client'):
 762                 client = r['Client']
 763                 break
 764         if not client:
 765             die("could not get client spec")
 766         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 767         for r in results:
 768             if r.has_key('change'):
 769                 return r['change']
 770         die("Could not get changelist number for last submit - cannot patch up user details")
 771
 772     def modifyChangelistUser(self, changelist, newUser):
 773         # fixup the user field of a changelist after it has been submitted.
 774         changes = p4CmdList("change -o %s" % changelist)
 775         if len(changes) != 1:
 776             die("Bad output from p4 change modifying %s to user %s" %
 777                 (changelist, newUser))
 778
 779         c = changes[0]
 780         if c['User'] == newUser: return   # nothing to do
 781         c['User'] = newUser
 782         input = marshal.dumps(c)
 783
 784         result = p4CmdList("change -f -i", stdin=input)
 785         for r in result:
 786             if r.has_key('code'):
 787                 if r['code'] == 'error':
 788                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 789             if r.has_key('data'):
 790                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 791                 return
 792         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 793
 794     def canChangeChangelists(self):
 795         # check to see if we have p4 admin or super-user permissions, either of
 796         # which are required to modify changelists.
 797         results = p4CmdList("protects %s" % self.depotPath)
 798         for r in results:
 799             if r.has_key('perm'):
 800                 if r['perm'] == 'admin':
 801                     return 1
 802                 if r['perm'] == 'super':
 803                     return 1
 804         return 0
 805
 806     def p4UserId(self):
 807         if self.myP4UserId:
 808             return self.myP4UserId
 809
 810         results = p4CmdList("user -o")
 811         for r in results:
 812             if r.has_key('User'):
 813                 self.myP4UserId = r['User']
 814                 return r['User']
 815         die("Could not find your p4 user id")
 816
 817     def p4UserIsMe(self, p4User):
 818         # return True if the given p4 user is actually me
 819         me = self.p4UserId()
 820         if not p4User or p4User != me:
 821             return False
 822         else:
 823             return True
 824
 825     def prepareSubmitTemplate(self):
 826         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 827         template = ""
 828         inFilesSection = False
 829         for line in p4_read_pipe_lines(['change', '-o']):
 830             if line.endswith("\r\n"):
 831                 line = line[:-2] + "\n"
 832             if inFilesSection:
 833                 if line.startswith("\t"):
 834                     # path starts and ends with a tab
 835                     path = line[1:]
 836                     lastTab = path.rfind("\t")
 837                     if lastTab != -1:
 838                         path = path[:lastTab]
 839                         if not p4PathStartsWith(path, self.depotPath):
 840                             continue
 841                 else:
 842                     inFilesSection = False
 843             else:
 844                 if line.startswith("Files:"):
 845                     inFilesSection = True
 846
 847             template += line
 848
 849         return template
 850
 851     def edit_template(self, template_file):
 852         """Invoke the editor to let the user change the submission
 853            message.  Return true if okay to continue with the submit."""
 854
 855         # if configured to skip the editing part, just submit
 856         if gitConfig("git-p4.skipSubmitEdit") == "true":
 857             return True
 858
 859         # look at the modification time, to check later if the user saved
 860         # the file
 861         mtime = os.stat(template_file).st_mtime
 862
 863         # invoke the editor
 864         if os.environ.has_key("P4EDITOR"):
 865             editor = os.environ.get("P4EDITOR")
 866         else:
 867             editor = read_pipe("git var GIT_EDITOR").strip()
 868         system(editor + " " + template_file)
 869
 870         # If the file was not saved, prompt to see if this patch should
 871         # be skipped.  But skip this verification step if configured so.
 872         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 873             return True
 874
 875         if os.stat(template_file).st_mtime <= mtime:
 876             while True:
 877                 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 878                 if response == 'y':
 879                     return True
 880                 if response == 'n':
 881                     return False
 882
 883     def applyCommit(self, id):
 884         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 885
 886         (p4User, gitEmail) = self.p4UserForCommit(id)
 887
 888         if not self.detectRenames:
 889             # If not explicitly set check the config variable
 890             self.detectRenames = gitConfig("git-p4.detectRenames")
 891
 892         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 893             diffOpts = ""
 894         elif self.detectRenames.lower() == "true":
 895             diffOpts = "-M"
 896         else:
 897             diffOpts = "-M%s" % self.detectRenames
 898
 899         detectCopies = gitConfig("git-p4.detectCopies")
 900         if detectCopies.lower() == "true":
 901             diffOpts += " -C"
 902         elif detectCopies != "" and detectCopies.lower() != "false":
 903             diffOpts += " -C%s" % detectCopies
 904
 905         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 906             diffOpts += " --find-copies-harder"
 907
 908         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 909         filesToAdd = set()
 910         filesToDelete = set()
 911         editedFiles = set()
 912         filesToChangeExecBit = {}
 913         for line in diff:
 914             diff = parseDiffTreeEntry(line)
 915             modifier = diff['status']
 916             path = diff['src']
 917             if modifier == "M":
 918                 p4_edit(path)
 919                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 920                     filesToChangeExecBit[path] = diff['dst_mode']
 921                 editedFiles.add(path)
 922             elif modifier == "A":
 923                 filesToAdd.add(path)
 924                 filesToChangeExecBit[path] = diff['dst_mode']
 925                 if path in filesToDelete:
 926                     filesToDelete.remove(path)
 927             elif modifier == "D":
 928                 filesToDelete.add(path)
 929                 if path in filesToAdd:
 930                     filesToAdd.remove(path)
 931             elif modifier == "C":
 932                 src, dest = diff['src'], diff['dst']
 933                 p4_integrate(src, dest)
 934                 if diff['src_sha1'] != diff['dst_sha1']:
 935                     p4_edit(dest)
 936                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 937                     p4_edit(dest)
 938                     filesToChangeExecBit[dest] = diff['dst_mode']
 939                 os.unlink(dest)
 940                 editedFiles.add(dest)
 941             elif modifier == "R":
 942                 src, dest = diff['src'], diff['dst']
 943                 p4_integrate(src, dest)
 944                 if diff['src_sha1'] != diff['dst_sha1']:
 945                     p4_edit(dest)
 946                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 947                     p4_edit(dest)
 948                     filesToChangeExecBit[dest] = diff['dst_mode']
 949                 os.unlink(dest)
 950                 editedFiles.add(dest)
 951                 filesToDelete.add(src)
 952             else:
 953                 die("unknown modifier %s for %s" % (modifier, path))
 954
 955         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 956         patchcmd = diffcmd + " | git apply "
 957         tryPatchCmd = patchcmd + "--check -"
 958         applyPatchCmd = patchcmd + "--check --apply -"
 959
 960         if os.system(tryPatchCmd) != 0:
 961             print "Unfortunately applying the change failed!"
 962             print "What do you want to do?"
 963             response = "x"
 964             while response != "s" and response != "a" and response != "w":
 965                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 966                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 967             if response == "s":
 968                 print "Skipping! Good luck with the next patches..."
 969                 for f in editedFiles:
 970                     p4_revert(f)
 971                 for f in filesToAdd:
 972                     os.remove(f)
 973                 return
 974             elif response == "a":
 975                 os.system(applyPatchCmd)
 976                 if len(filesToAdd) > 0:
 977                     print "You may also want to call p4 add on the following files:"
 978                     print " ".join(filesToAdd)
 979                 if len(filesToDelete):
 980                     print "The following files should be scheduled for deletion with p4 delete:"
 981                     print " ".join(filesToDelete)
 982                 die("Please resolve and submit the conflict manually and "
 983                     + "continue afterwards with git-p4 submit --continue")
 984             elif response == "w":
 985                 system(diffcmd + " > patch.txt")
 986                 print "Patch saved to patch.txt in %s !" % self.clientPath
 987                 die("Please resolve and submit the conflict manually and "
 988                     "continue afterwards with git-p4 submit --continue")
 989
 990         system(applyPatchCmd)
 991
 992         for f in filesToAdd:
 993             p4_add(f)
 994         for f in filesToDelete:
 995             p4_revert(f)
 996             p4_delete(f)
 997
 998         # Set/clear executable bits
 999         for f in filesToChangeExecBit.keys():
1000             mode = filesToChangeExecBit[f]
1001             setP4ExecBit(f, mode)
1002
1003         logMessage = extractLogMessageFromGitCommit(id)
1004         logMessage = logMessage.strip()
1005
1006         template = self.prepareSubmitTemplate()
1007
1008         if self.interactive:
1009             submitTemplate = self.prepareLogMessage(template, logMessage)
1010
1011             if self.preserveUser:
1012                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1013
1014             if os.environ.has_key("P4DIFF"):
1015                 del(os.environ["P4DIFF"])
1016             diff = ""
1017             for editedFile in editedFiles:
1018                 diff += p4_read_pipe(['diff', '-du', editedFile])
1019
1020             newdiff = ""
1021             for newFile in filesToAdd:
1022                 newdiff += "==== new file ====\n"
1023                 newdiff += "--- /dev/null\n"
1024                 newdiff += "+++ %s\n" % newFile
1025                 f = open(newFile, "r")
1026                 for line in f.readlines():
1027                     newdiff += "+" + line
1028                 f.close()
1029
1030             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1031                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1032                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1033                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1034
1035             separatorLine = "######## everything below this line is just the diff #######\n"
1036
1037             (handle, fileName) = tempfile.mkstemp()
1038             tmpFile = os.fdopen(handle, "w+")
1039             if self.isWindows:
1040                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1041                 separatorLine = separatorLine.replace("\n", "\r\n")
1042                 newdiff = newdiff.replace("\n", "\r\n")
1043             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1044             tmpFile.close()
1045
1046             if self.edit_template(fileName):
1047                 # read the edited message and submit
1048                 tmpFile = open(fileName, "rb")
1049                 message = tmpFile.read()
1050                 tmpFile.close()
1051                 submitTemplate = message[:message.index(separatorLine)]
1052                 if self.isWindows:
1053                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1054                 p4_write_pipe(['submit', '-i'], submitTemplate)
1055
1056                 if self.preserveUser:
1057                     if p4User:
1058                         # Get last changelist number. Cannot easily get it from
1059                         # the submit command output as the output is
1060                         # unmarshalled.
1061                         changelist = self.lastP4Changelist()
1062                         self.modifyChangelistUser(changelist, p4User)
1063             else:
1064                 # skip this patch
1065                 for f in editedFiles:
1066                     p4_revert(f)
1067                 for f in filesToAdd:
1068                     p4_revert(f)
1069                     os.remove(f)
1070
1071             os.remove(fileName)
1072         else:
1073             fileName = "submit.txt"
1074             file = open(fileName, "w+")
1075             file.write(self.prepareLogMessage(template, logMessage))
1076             file.close()
1077             print ("Perforce submit template written as %s. "
1078                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1079                    % (fileName, fileName))
1080
1081     def run(self, args):
1082         if len(args) == 0:
1083             self.master = currentGitBranch()
1084             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1085                 die("Detecting current git branch failed!")
1086         elif len(args) == 1:
1087             self.master = args[0]
1088         else:
1089             return False
1090
1091         allowSubmit = gitConfig("git-p4.allowSubmit")
1092         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1093             die("%s is not in git-p4.allowSubmit" % self.master)
1094
1095         [upstream, settings] = findUpstreamBranchPoint()
1096         self.depotPath = settings['depot-paths'][0]
1097         if len(self.origin) == 0:
1098             self.origin = upstream
1099
1100         if self.preserveUser:
1101             if not self.canChangeChangelists():
1102                 die("Cannot preserve user names without p4 super-user or admin permissions")
1103
1104         if self.verbose:
1105             print "Origin branch is " + self.origin
1106
1107         if len(self.depotPath) == 0:
1108             print "Internal error: cannot locate perforce depot path from existing branches"
1109             sys.exit(128)
1110
1111         self.clientPath = p4Where(self.depotPath)
1112
1113         if len(self.clientPath) == 0:
1114             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1115             sys.exit(128)
1116
1117         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1118         self.oldWorkingDirectory = os.getcwd()
1119
1120         # ensure the clientPath exists
1121         if not os.path.exists(self.clientPath):
1122             os.makedirs(self.clientPath)
1123
1124         chdir(self.clientPath)
1125         print "Synchronizing p4 checkout..."
1126         p4_sync("...")
1127         self.check()
1128
1129         commits = []
1130         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1131             commits.append(line.strip())
1132         commits.reverse()
1133
1134         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1135             self.checkAuthorship = False
1136         else:
1137             self.checkAuthorship = True
1138
1139         if self.preserveUser:
1140             self.checkValidP4Users(commits)
1141
1142         while len(commits) > 0:
1143             commit = commits[0]
1144             commits = commits[1:]
1145             self.applyCommit(commit)
1146             if not self.interactive:
1147                 break
1148
1149         if len(commits) == 0:
1150             print "All changes applied!"
1151             chdir(self.oldWorkingDirectory)
1152
1153             sync = P4Sync()
1154             sync.run([])
1155
1156             rebase = P4Rebase()
1157             rebase.rebase()
1158
1159         return True
1160
1161 class P4Sync(Command, P4UserMap):
1162     delete_actions = ( "delete", "move/delete", "purge" )
1163
1164     def __init__(self):
1165         Command.__init__(self)
1166         P4UserMap.__init__(self)
1167         self.options = [
1168                 optparse.make_option("--branch", dest="branch"),
1169                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1170                 optparse.make_option("--changesfile", dest="changesFile"),
1171                 optparse.make_option("--silent", dest="silent", action="store_true"),
1172                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1173                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1174                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1175                                      help="Import into refs/heads/ , not refs/remotes"),
1176                 optparse.make_option("--max-changes", dest="maxChanges"),
1177                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1178                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1179                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1180                                      help="Only sync files that are included in the Perforce Client Spec")
1181         ]
1182         self.description = """Imports from Perforce into a git repository.\n
1183     example:
1184     //depot/my/project/ -- to import the current head
1185     //depot/my/project/@all -- to import everything
1186     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1187
1188     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1189
1190         self.usage += " //depot/path[@revRange]"
1191         self.silent = False
1192         self.createdBranches = set()
1193         self.committedChanges = set()
1194         self.branch = ""
1195         self.detectBranches = False
1196         self.detectLabels = False
1197         self.changesFile = ""
1198         self.syncWithOrigin = True
1199         self.verbose = False
1200         self.importIntoRemotes = True
1201         self.maxChanges = ""
1202         self.isWindows = (platform.system() == "Windows")
1203         self.keepRepoPath = False
1204         self.depotPaths = None
1205         self.p4BranchesInGit = []
1206         self.cloneExclude = []
1207         self.useClientSpec = False
1208         self.clientSpecDirs = []
1209
1210         if gitConfig("git-p4.syncFromOrigin") == "false":
1211             self.syncWithOrigin = False
1212
1213     #
1214     # P4 wildcards are not allowed in filenames.  P4 complains
1215     # if you simply add them, but you can force it with "-f", in
1216     # which case it translates them into %xx encoding internally.
1217     # Search for and fix just these four characters.  Do % last so
1218     # that fixing it does not inadvertently create new %-escapes.
1219     #
1220     def wildcard_decode(self, path):
1221         # Cannot have * in a filename in windows; untested as to
1222         # what p4 would do in such a case.
1223         if not self.isWindows:
1224             path = path.replace("%2A", "*")
1225         path = path.replace("%23", "#") \
1226                    .replace("%40", "@") \
1227                    .replace("%25", "%")
1228         return path
1229
1230     def extractFilesFromCommit(self, commit):
1231         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1232                              for path in self.cloneExclude]
1233         files = []
1234         fnum = 0
1235         while commit.has_key("depotFile%s" % fnum):
1236             path =  commit["depotFile%s" % fnum]
1237
1238             if [p for p in self.cloneExclude
1239                 if p4PathStartsWith(path, p)]:
1240                 found = False
1241             else:
1242                 found = [p for p in self.depotPaths
1243                          if p4PathStartsWith(path, p)]
1244             if not found:
1245                 fnum = fnum + 1
1246                 continue
1247
1248             file = {}
1249             file["path"] = path
1250             file["rev"] = commit["rev%s" % fnum]
1251             file["action"] = commit["action%s" % fnum]
1252             file["type"] = commit["type%s" % fnum]
1253             files.append(file)
1254             fnum = fnum + 1
1255         return files
1256
1257     def stripRepoPath(self, path, prefixes):
1258         if self.useClientSpec:
1259
1260             # if using the client spec, we use the output directory
1261             # specified in the client.  For example, a view
1262             #   //depot/foo/branch/... //client/branch/foo/...
1263             # will end up putting all foo/branch files into
1264             #  branch/foo/
1265             for val in self.clientSpecDirs:
1266                 if path.startswith(val[0]):
1267                     # replace the depot path with the client path
1268                     path = path.replace(val[0], val[1][1])
1269                     # now strip out the client (//client/...)
1270                     path = re.sub("^(//[^/]+/)", '', path)
1271                     # the rest is all path
1272                     return path
1273
1274         if self.keepRepoPath:
1275             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1276
1277         for p in prefixes:
1278             if p4PathStartsWith(path, p):
1279                 path = path[len(p):]
1280
1281         return path
1282
1283     def splitFilesIntoBranches(self, commit):
1284         branches = {}
1285         fnum = 0
1286         while commit.has_key("depotFile%s" % fnum):
1287             path =  commit["depotFile%s" % fnum]
1288             found = [p for p in self.depotPaths
1289                      if p4PathStartsWith(path, p)]
1290             if not found:
1291                 fnum = fnum + 1
1292                 continue
1293
1294             file = {}
1295             file["path"] = path
1296             file["rev"] = commit["rev%s" % fnum]
1297             file["action"] = commit["action%s" % fnum]
1298             file["type"] = commit["type%s" % fnum]
1299             fnum = fnum + 1
1300
1301             relPath = self.stripRepoPath(path, self.depotPaths)
1302
1303             for branch in self.knownBranches.keys():
1304
1305                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1306                 if relPath.startswith(branch + "/"):
1307                     if branch not in branches:
1308                         branches[branch] = []
1309                     branches[branch].append(file)
1310                     break
1311
1312         return branches
1313
1314     # output one file from the P4 stream
1315     # - helper for streamP4Files
1316
1317     def streamOneP4File(self, file, contents):
1318         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1319         relPath = self.wildcard_decode(relPath)
1320         if verbose:
1321             sys.stderr.write("%s\n" % relPath)
1322
1323         (type_base, type_mods) = split_p4_type(file["type"])
1324
1325         git_mode = "100644"
1326         if "x" in type_mods:
1327             git_mode = "100755"
1328         if type_base == "symlink":
1329             git_mode = "120000"
1330             # p4 print on a symlink contains "target\n"; remove the newline
1331             data = ''.join(contents)
1332             contents = [data[:-1]]
1333
1334         if type_base == "utf16":
1335             # p4 delivers different text in the python output to -G
1336             # than it does when using "print -o", or normal p4 client
1337             # operations.  utf16 is converted to ascii or utf8, perhaps.
1338             # But ascii text saved as -t utf16 is completely mangled.
1339             # Invoke print -o to get the real contents.
1340             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1341             contents = [ text ]
1342
1343         if type_base == "apple":
1344             # Apple filetype files will be streamed as a concatenation of
1345             # its appledouble header and the contents.  This is useless
1346             # on both macs and non-macs.  If using "print -q -o xx", it
1347             # will create "xx" with the data, and "%xx" with the header.
1348             # This is also not very useful.
1349             #
1350             # Ideally, someday, this script can learn how to generate
1351             # appledouble files directly and import those to git, but
1352             # non-mac machines can never find a use for apple filetype.
1353             print "\nIgnoring apple filetype file %s" % file['depotFile']
1354             return
1355
1356         # Perhaps windows wants unicode, utf16 newlines translated too;
1357         # but this is not doing it.
1358         if self.isWindows and type_base == "text":
1359             mangled = []
1360             for data in contents:
1361                 data = data.replace("\r\n", "\n")
1362                 mangled.append(data)
1363             contents = mangled
1364
1365         # Note that we do not try to de-mangle keywords on utf16 files,
1366         # even though in theory somebody may want that.
1367         if type_base in ("text", "unicode", "binary"):
1368             if "ko" in type_mods:
1369                 text = ''.join(contents)
1370                 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1371                 contents = [ text ]
1372             elif "k" in type_mods:
1373                 text = ''.join(contents)
1374                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1375                 contents = [ text ]
1376
1377         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1378
1379         # total length...
1380         length = 0
1381         for d in contents:
1382             length = length + len(d)
1383
1384         self.gitStream.write("data %d\n" % length)
1385         for d in contents:
1386             self.gitStream.write(d)
1387         self.gitStream.write("\n")
1388
1389     def streamOneP4Deletion(self, file):
1390         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1391         if verbose:
1392             sys.stderr.write("delete %s\n" % relPath)
1393         self.gitStream.write("D %s\n" % relPath)
1394
1395     # handle another chunk of streaming data
1396     def streamP4FilesCb(self, marshalled):
1397
1398         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1399             # start of a new file - output the old one first
1400             self.streamOneP4File(self.stream_file, self.stream_contents)
1401             self.stream_file = {}
1402             self.stream_contents = []
1403             self.stream_have_file_info = False
1404
1405         # pick up the new file information... for the
1406         # 'data' field we need to append to our array
1407         for k in marshalled.keys():
1408             if k == 'data':
1409                 self.stream_contents.append(marshalled['data'])
1410             else:
1411                 self.stream_file[k] = marshalled[k]
1412
1413         self.stream_have_file_info = True
1414
1415     # Stream directly from "p4 files" into "git fast-import"
1416     def streamP4Files(self, files):
1417         filesForCommit = []
1418         filesToRead = []
1419         filesToDelete = []
1420
1421         for f in files:
1422             includeFile = True
1423             for val in self.clientSpecDirs:
1424                 if f['path'].startswith(val[0]):
1425                     if val[1][0] <= 0:
1426                         includeFile = False
1427                     break
1428
1429             if includeFile:
1430                 filesForCommit.append(f)
1431                 if f['action'] in self.delete_actions:
1432                     filesToDelete.append(f)
1433                 else:
1434                     filesToRead.append(f)
1435
1436         # deleted files...
1437         for f in filesToDelete:
1438             self.streamOneP4Deletion(f)
1439
1440         if len(filesToRead) > 0:
1441             self.stream_file = {}
1442             self.stream_contents = []
1443             self.stream_have_file_info = False
1444
1445             # curry self argument
1446             def streamP4FilesCbSelf(entry):
1447                 self.streamP4FilesCb(entry)
1448
1449             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1450
1451             p4CmdList(["-x", "-", "print"],
1452                       stdin=fileArgs,
1453                       cb=streamP4FilesCbSelf)
1454
1455             # do the last chunk
1456             if self.stream_file.has_key('depotFile'):
1457                 self.streamOneP4File(self.stream_file, self.stream_contents)
1458
1459     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1460         epoch = details["time"]
1461         author = details["user"]
1462         self.branchPrefixes = branchPrefixes
1463
1464         if self.verbose:
1465             print "commit into %s" % branch
1466
1467         # start with reading files; if that fails, we should not
1468         # create a commit.
1469         new_files = []
1470         for f in files:
1471             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1472                 new_files.append (f)
1473             else:
1474                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1475
1476         self.gitStream.write("commit %s\n" % branch)
1477 #        gitStream.write("mark :%s\n" % details["change"])
1478         self.committedChanges.add(int(details["change"]))
1479         committer = ""
1480         if author not in self.users:
1481             self.getUserMapFromPerforceServer()
1482         if author in self.users:
1483             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1484         else:
1485             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1486
1487         self.gitStream.write("committer %s\n" % committer)
1488
1489         self.gitStream.write("data <<EOT\n")
1490         self.gitStream.write(details["desc"])
1491         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1492                              % (','.join (branchPrefixes), details["change"]))
1493         if len(details['options']) > 0:
1494             self.gitStream.write(": options = %s" % details['options'])
1495         self.gitStream.write("]\nEOT\n\n")
1496
1497         if len(parent) > 0:
1498             if self.verbose:
1499                 print "parent %s" % parent
1500             self.gitStream.write("from %s\n" % parent)
1501
1502         self.streamP4Files(new_files)
1503         self.gitStream.write("\n")
1504
1505         change = int(details["change"])
1506
1507         if self.labels.has_key(change):
1508             label = self.labels[change]
1509             labelDetails = label[0]
1510             labelRevisions = label[1]
1511             if self.verbose:
1512                 print "Change %s is labelled %s" % (change, labelDetails)
1513
1514             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1515                                                     for p in branchPrefixes])
1516
1517             if len(files) == len(labelRevisions):
1518
1519                 cleanedFiles = {}
1520                 for info in files:
1521                     if info["action"] in self.delete_actions:
1522                         continue
1523                     cleanedFiles[info["depotFile"]] = info["rev"]
1524
1525                 if cleanedFiles == labelRevisions:
1526                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1527                     self.gitStream.write("from %s\n" % branch)
1528
1529                     owner = labelDetails["Owner"]
1530                     tagger = ""
1531                     if author in self.users:
1532                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1533                     else:
1534                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1535                     self.gitStream.write("tagger %s\n" % tagger)
1536                     self.gitStream.write("data <<EOT\n")
1537                     self.gitStream.write(labelDetails["Description"])
1538                     self.gitStream.write("EOT\n\n")
1539
1540                 else:
1541                     if not self.silent:
1542                         print ("Tag %s does not match with change %s: files do not match."
1543                                % (labelDetails["label"], change))
1544
1545             else:
1546                 if not self.silent:
1547                     print ("Tag %s does not match with change %s: file count is different."
1548                            % (labelDetails["label"], change))
1549
1550     def getLabels(self):
1551         self.labels = {}
1552
1553         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1554         if len(l) > 0 and not self.silent:
1555             print "Finding files belonging to labels in %s" % `self.depotPaths`
1556
1557         for output in l:
1558             label = output["label"]
1559             revisions = {}
1560             newestChange = 0
1561             if self.verbose:
1562                 print "Querying files for label %s" % label
1563             for file in p4CmdList(["files"] +
1564                                       ["%s...@%s" % (p, label)
1565                                           for p in self.depotPaths]):
1566                 revisions[file["depotFile"]] = file["rev"]
1567                 change = int(file["change"])
1568                 if change > newestChange:
1569                     newestChange = change
1570
1571             self.labels[newestChange] = [output, revisions]
1572
1573         if self.verbose:
1574             print "Label changes: %s" % self.labels.keys()
1575
1576     def guessProjectName(self):
1577         for p in self.depotPaths:
1578             if p.endswith("/"):
1579                 p = p[:-1]
1580             p = p[p.strip().rfind("/") + 1:]
1581             if not p.endswith("/"):
1582                p += "/"
1583             return p
1584
1585     def getBranchMapping(self):
1586         lostAndFoundBranches = set()
1587
1588         user = gitConfig("git-p4.branchUser")
1589         if len(user) > 0:
1590             command = "branches -u %s" % user
1591         else:
1592             command = "branches"
1593
1594         for info in p4CmdList(command):
1595             details = p4Cmd("branch -o %s" % info["branch"])
1596             viewIdx = 0
1597             while details.has_key("View%s" % viewIdx):
1598                 paths = details["View%s" % viewIdx].split(" ")
1599                 viewIdx = viewIdx + 1
1600                 # require standard //depot/foo/... //depot/bar/... mapping
1601                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1602                     continue
1603                 source = paths[0]
1604                 destination = paths[1]
1605                 ## HACK
1606                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1607                     source = source[len(self.depotPaths[0]):-4]
1608                     destination = destination[len(self.depotPaths[0]):-4]
1609
1610                     if destination in self.knownBranches:
1611                         if not self.silent:
1612                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1613                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1614                         continue
1615
1616                     self.knownBranches[destination] = source
1617
1618                     lostAndFoundBranches.discard(destination)
1619
1620                     if source not in self.knownBranches:
1621                         lostAndFoundBranches.add(source)
1622
1623         # Perforce does not strictly require branches to be defined, so we also
1624         # check git config for a branch list.
1625         #
1626         # Example of branch definition in git config file:
1627         # [git-p4]
1628         #   branchList=main:branchA
1629         #   branchList=main:branchB
1630         #   branchList=branchA:branchC
1631         configBranches = gitConfigList("git-p4.branchList")
1632         for branch in configBranches:
1633             if branch:
1634                 (source, destination) = branch.split(":")
1635                 self.knownBranches[destination] = source
1636
1637                 lostAndFoundBranches.discard(destination)
1638
1639                 if source not in self.knownBranches:
1640                     lostAndFoundBranches.add(source)
1641
1642
1643         for branch in lostAndFoundBranches:
1644             self.knownBranches[branch] = branch
1645
1646     def getBranchMappingFromGitBranches(self):
1647         branches = p4BranchesInGit(self.importIntoRemotes)
1648         for branch in branches.keys():
1649             if branch == "master":
1650                 branch = "main"
1651             else:
1652                 branch = branch[len(self.projectName):]
1653             self.knownBranches[branch] = branch
1654
1655     def listExistingP4GitBranches(self):
1656         # branches holds mapping from name to commit
1657         branches = p4BranchesInGit(self.importIntoRemotes)
1658         self.p4BranchesInGit = branches.keys()
1659         for branch in branches.keys():
1660             self.initialParents[self.refPrefix + branch] = branches[branch]
1661
1662     def updateOptionDict(self, d):
1663         option_keys = {}
1664         if self.keepRepoPath:
1665             option_keys['keepRepoPath'] = 1
1666
1667         d["options"] = ' '.join(sorted(option_keys.keys()))
1668
1669     def readOptions(self, d):
1670         self.keepRepoPath = (d.has_key('options')
1671                              and ('keepRepoPath' in d['options']))
1672
1673     def gitRefForBranch(self, branch):
1674         if branch == "main":
1675             return self.refPrefix + "master"
1676
1677         if len(branch) <= 0:
1678             return branch
1679
1680         return self.refPrefix + self.projectName + branch
1681
1682     def gitCommitByP4Change(self, ref, change):
1683         if self.verbose:
1684             print "looking in ref " + ref + " for change %s using bisect..." % change
1685
1686         earliestCommit = ""
1687         latestCommit = parseRevision(ref)
1688
1689         while True:
1690             if self.verbose:
1691                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1692             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1693             if len(next) == 0:
1694                 if self.verbose:
1695                     print "argh"
1696                 return ""
1697             log = extractLogMessageFromGitCommit(next)
1698             settings = extractSettingsGitLog(log)
1699             currentChange = int(settings['change'])
1700             if self.verbose:
1701                 print "current change %s" % currentChange
1702
1703             if currentChange == change:
1704                 if self.verbose:
1705                     print "found %s" % next
1706                 return next
1707
1708             if currentChange < change:
1709                 earliestCommit = "^%s" % next
1710             else:
1711                 latestCommit = "%s" % next
1712
1713         return ""
1714
1715     def importNewBranch(self, branch, maxChange):
1716         # make fast-import flush all changes to disk and update the refs using the checkpoint
1717         # command so that we can try to find the branch parent in the git history
1718         self.gitStream.write("checkpoint\n\n");
1719         self.gitStream.flush();
1720         branchPrefix = self.depotPaths[0] + branch + "/"
1721         range = "@1,%s" % maxChange
1722         #print "prefix" + branchPrefix
1723         changes = p4ChangesForPaths([branchPrefix], range)
1724         if len(changes) <= 0:
1725             return False
1726         firstChange = changes[0]
1727         #print "first change in branch: %s" % firstChange
1728         sourceBranch = self.knownBranches[branch]
1729         sourceDepotPath = self.depotPaths[0] + sourceBranch
1730         sourceRef = self.gitRefForBranch(sourceBranch)
1731         #print "source " + sourceBranch
1732
1733         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1734         #print "branch parent: %s" % branchParentChange
1735         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1736         if len(gitParent) > 0:
1737             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1738             #print "parent git commit: %s" % gitParent
1739
1740         self.importChanges(changes)
1741         return True
1742
1743     def importChanges(self, changes):
1744         cnt = 1
1745         for change in changes:
1746             description = p4Cmd("describe %s" % change)
1747             self.updateOptionDict(description)
1748
1749             if not self.silent:
1750                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1751                 sys.stdout.flush()
1752             cnt = cnt + 1
1753
1754             try:
1755                 if self.detectBranches:
1756                     branches = self.splitFilesIntoBranches(description)
1757                     for branch in branches.keys():
1758                         ## HACK  --hwn
1759                         branchPrefix = self.depotPaths[0] + branch + "/"
1760
1761                         parent = ""
1762
1763                         filesForCommit = branches[branch]
1764
1765                         if self.verbose:
1766                             print "branch is %s" % branch
1767
1768                         self.updatedBranches.add(branch)
1769
1770                         if branch not in self.createdBranches:
1771                             self.createdBranches.add(branch)
1772                             parent = self.knownBranches[branch]
1773                             if parent == branch:
1774                                 parent = ""
1775                             else:
1776                                 fullBranch = self.projectName + branch
1777                                 if fullBranch not in self.p4BranchesInGit:
1778                                     if not self.silent:
1779                                         print("\n    Importing new branch %s" % fullBranch);
1780                                     if self.importNewBranch(branch, change - 1):
1781                                         parent = ""
1782                                         self.p4BranchesInGit.append(fullBranch)
1783                                     if not self.silent:
1784                                         print("\n    Resuming with change %s" % change);
1785
1786                                 if self.verbose:
1787                                     print "parent determined through known branches: %s" % parent
1788
1789                         branch = self.gitRefForBranch(branch)
1790                         parent = self.gitRefForBranch(parent)
1791
1792                         if self.verbose:
1793                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1794
1795                         if len(parent) == 0 and branch in self.initialParents:
1796                             parent = self.initialParents[branch]
1797                             del self.initialParents[branch]
1798
1799                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1800                 else:
1801                     files = self.extractFilesFromCommit(description)
1802                     self.commit(description, files, self.branch, self.depotPaths,
1803                                 self.initialParent)
1804                     self.initialParent = ""
1805             except IOError:
1806                 print self.gitError.read()
1807                 sys.exit(1)
1808
1809     def importHeadRevision(self, revision):
1810         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1811
1812         details = {}
1813         details["user"] = "git perforce import user"
1814         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1815                            % (' '.join(self.depotPaths), revision))
1816         details["change"] = revision
1817         newestRevision = 0
1818
1819         fileCnt = 0
1820         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
1821
1822         for info in p4CmdList(["files"] + fileArgs):
1823
1824             if 'code' in info and info['code'] == 'error':
1825                 sys.stderr.write("p4 returned an error: %s\n"
1826                                  % info['data'])
1827                 if info['data'].find("must refer to client") >= 0:
1828                     sys.stderr.write("This particular p4 error is misleading.\n")
1829                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1830                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1831                 sys.exit(1)
1832             if 'p4ExitCode' in info:
1833                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1834                 sys.exit(1)
1835
1836
1837             change = int(info["change"])
1838             if change > newestRevision:
1839                 newestRevision = change
1840
1841             if info["action"] in self.delete_actions:
1842                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1843                 #fileCnt = fileCnt + 1
1844                 continue
1845
1846             for prop in ["depotFile", "rev", "action", "type" ]:
1847                 details["%s%s" % (prop, fileCnt)] = info[prop]
1848
1849             fileCnt = fileCnt + 1
1850
1851         details["change"] = newestRevision
1852
1853         # Use time from top-most change so that all git-p4 clones of
1854         # the same p4 repo have the same commit SHA1s.
1855         res = p4CmdList("describe -s %d" % newestRevision)
1856         newestTime = None
1857         for r in res:
1858             if r.has_key('time'):
1859                 newestTime = int(r['time'])
1860         if newestTime is None:
1861             die("\"describe -s\" on newest change %d did not give a time")
1862         details["time"] = newestTime
1863
1864         self.updateOptionDict(details)
1865         try:
1866             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1867         except IOError:
1868             print "IO error with git fast-import. Is your git version recent enough?"
1869             print self.gitError.read()
1870
1871
1872     def getClientSpec(self):
1873         specList = p4CmdList( "client -o" )
1874         temp = {}
1875         for entry in specList:
1876             for k,v in entry.iteritems():
1877                 if k.startswith("View"):
1878
1879                     # p4 has these %%1 to %%9 arguments in specs to
1880                     # reorder paths; which we can't handle (yet :)
1881                     if re.match('%%\d', v) != None:
1882                         print "Sorry, can't handle %%n arguments in client specs"
1883                         sys.exit(1)
1884
1885                     if v.startswith('"'):
1886                         start = 1
1887                     else:
1888                         start = 0
1889                     index = v.find("...")
1890
1891                     # save the "client view"; i.e the RHS of the view
1892                     # line that tells the client where to put the
1893                     # files for this view.
1894                     cv = v[index+3:].strip() # +3 to remove previous '...'
1895
1896                     # if the client view doesn't end with a
1897                     # ... wildcard, then we're going to mess up the
1898                     # output directory, so fail gracefully.
1899                     if not cv.endswith('...'):
1900                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1901                         sys.exit(1)
1902                     cv=cv[:-3]
1903
1904                     # now save the view; +index means included, -index
1905                     # means it should be filtered out.
1906                     v = v[start:index]
1907                     if v.startswith("-"):
1908                         v = v[1:]
1909                         include = -len(v)
1910                     else:
1911                         include = len(v)
1912
1913                     temp[v] = (include, cv)
1914
1915         self.clientSpecDirs = temp.items()
1916         self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1917
1918     def run(self, args):
1919         self.depotPaths = []
1920         self.changeRange = ""
1921         self.initialParent = ""
1922         self.previousDepotPaths = []
1923
1924         # map from branch depot path to parent branch
1925         self.knownBranches = {}
1926         self.initialParents = {}
1927         self.hasOrigin = originP4BranchesExist()
1928         if not self.syncWithOrigin:
1929             self.hasOrigin = False
1930
1931         if self.importIntoRemotes:
1932             self.refPrefix = "refs/remotes/p4/"
1933         else:
1934             self.refPrefix = "refs/heads/p4/"
1935
1936         if self.syncWithOrigin and self.hasOrigin:
1937             if not self.silent:
1938                 print "Syncing with origin first by calling git fetch origin"
1939             system("git fetch origin")
1940
1941         if len(self.branch) == 0:
1942             self.branch = self.refPrefix + "master"
1943             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1944                 system("git update-ref %s refs/heads/p4" % self.branch)
1945                 system("git branch -D p4");
1946             # create it /after/ importing, when master exists
1947             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1948                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1949
1950         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1951             self.getClientSpec()
1952
1953         # TODO: should always look at previous commits,
1954         # merge with previous imports, if possible.
1955         if args == []:
1956             if self.hasOrigin:
1957                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1958             self.listExistingP4GitBranches()
1959
1960             if len(self.p4BranchesInGit) > 1:
1961                 if not self.silent:
1962                     print "Importing from/into multiple branches"
1963                 self.detectBranches = True
1964
1965             if self.verbose:
1966                 print "branches: %s" % self.p4BranchesInGit
1967
1968             p4Change = 0
1969             for branch in self.p4BranchesInGit:
1970                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1971
1972                 settings = extractSettingsGitLog(logMsg)
1973
1974                 self.readOptions(settings)
1975                 if (settings.has_key('depot-paths')
1976                     and settings.has_key ('change')):
1977                     change = int(settings['change']) + 1
1978                     p4Change = max(p4Change, change)
1979
1980                     depotPaths = sorted(settings['depot-paths'])
1981                     if self.previousDepotPaths == []:
1982                         self.previousDepotPaths = depotPaths
1983                     else:
1984                         paths = []
1985                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1986                             prev_list = prev.split("/")
1987                             cur_list = cur.split("/")
1988                             for i in range(0, min(len(cur_list), len(prev_list))):
1989                                 if cur_list[i] <> prev_list[i]:
1990                                     i = i - 1
1991                                     break
1992
1993                             paths.append ("/".join(cur_list[:i + 1]))
1994
1995                         self.previousDepotPaths = paths
1996
1997             if p4Change > 0:
1998                 self.depotPaths = sorted(self.previousDepotPaths)
1999                 self.changeRange = "@%s,#head" % p4Change
2000                 if not self.detectBranches:
2001                     self.initialParent = parseRevision(self.branch)
2002                 if not self.silent and not self.detectBranches:
2003                     print "Performing incremental import into %s git branch" % self.branch
2004
2005         if not self.branch.startswith("refs/"):
2006             self.branch = "refs/heads/" + self.branch
2007
2008         if len(args) == 0 and self.depotPaths:
2009             if not self.silent:
2010                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2011         else:
2012             if self.depotPaths and self.depotPaths != args:
2013                 print ("previous import used depot path %s and now %s was specified. "
2014                        "This doesn't work!" % (' '.join (self.depotPaths),
2015                                                ' '.join (args)))
2016                 sys.exit(1)
2017
2018             self.depotPaths = sorted(args)
2019
2020         revision = ""
2021         self.users = {}
2022
2023         newPaths = []
2024         for p in self.depotPaths:
2025             if p.find("@") != -1:
2026                 atIdx = p.index("@")
2027                 self.changeRange = p[atIdx:]
2028                 if self.changeRange == "@all":
2029                     self.changeRange = ""
2030                 elif ',' not in self.changeRange:
2031                     revision = self.changeRange
2032                     self.changeRange = ""
2033                 p = p[:atIdx]
2034             elif p.find("#") != -1:
2035                 hashIdx = p.index("#")
2036                 revision = p[hashIdx:]
2037                 p = p[:hashIdx]
2038             elif self.previousDepotPaths == []:
2039                 revision = "#head"
2040
2041             p = re.sub ("\.\.\.$", "", p)
2042             if not p.endswith("/"):
2043                 p += "/"
2044
2045             newPaths.append(p)
2046
2047         self.depotPaths = newPaths
2048
2049
2050         self.loadUserMapFromCache()
2051         self.labels = {}
2052         if self.detectLabels:
2053             self.getLabels();
2054
2055         if self.detectBranches:
2056             ## FIXME - what's a P4 projectName ?
2057             self.projectName = self.guessProjectName()
2058
2059             if self.hasOrigin:
2060                 self.getBranchMappingFromGitBranches()
2061             else:
2062                 self.getBranchMapping()
2063             if self.verbose:
2064                 print "p4-git branches: %s" % self.p4BranchesInGit
2065                 print "initial parents: %s" % self.initialParents
2066             for b in self.p4BranchesInGit:
2067                 if b != "master":
2068
2069                     ## FIXME
2070                     b = b[len(self.projectName):]
2071                 self.createdBranches.add(b)
2072
2073         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2074
2075         importProcess = subprocess.Popen(["git", "fast-import"],
2076                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2077                                          stderr=subprocess.PIPE);
2078         self.gitOutput = importProcess.stdout
2079         self.gitStream = importProcess.stdin
2080         self.gitError = importProcess.stderr
2081
2082         if revision:
2083             self.importHeadRevision(revision)
2084         else:
2085             changes = []
2086
2087             if len(self.changesFile) > 0:
2088                 output = open(self.changesFile).readlines()
2089                 changeSet = set()
2090                 for line in output:
2091                     changeSet.add(int(line))
2092
2093                 for change in changeSet:
2094                     changes.append(change)
2095
2096                 changes.sort()
2097             else:
2098                 # catch "git-p4 sync" with no new branches, in a repo that
2099                 # does not have any existing git-p4 branches
2100                 if len(args) == 0 and not self.p4BranchesInGit:
2101                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2102                 if self.verbose:
2103                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2104                                                               self.changeRange)
2105                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2106
2107                 if len(self.maxChanges) > 0:
2108                     changes = changes[:min(int(self.maxChanges), len(changes))]
2109
2110             if len(changes) == 0:
2111                 if not self.silent:
2112                     print "No changes to import!"
2113                 return True
2114
2115             if not self.silent and not self.detectBranches:
2116                 print "Import destination: %s" % self.branch
2117
2118             self.updatedBranches = set()
2119
2120             self.importChanges(changes)
2121
2122             if not self.silent:
2123                 print ""
2124                 if len(self.updatedBranches) > 0:
2125                     sys.stdout.write("Updated branches: ")
2126                     for b in self.updatedBranches:
2127                         sys.stdout.write("%s " % b)
2128                     sys.stdout.write("\n")
2129
2130         self.gitStream.close()
2131         if importProcess.wait() != 0:
2132             die("fast-import failed: %s" % self.gitError.read())
2133         self.gitOutput.close()
2134         self.gitError.close()
2135
2136         return True
2137
2138 class P4Rebase(Command):
2139     def __init__(self):
2140         Command.__init__(self)
2141         self.options = [ ]
2142         self.description = ("Fetches the latest revision from perforce and "
2143                             + "rebases the current work (branch) against it")
2144         self.verbose = False
2145
2146     def run(self, args):
2147         sync = P4Sync()
2148         sync.run([])
2149
2150         return self.rebase()
2151
2152     def rebase(self):
2153         if os.system("git update-index --refresh") != 0:
2154             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2155         if len(read_pipe("git diff-index HEAD --")) > 0:
2156             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2157
2158         [upstream, settings] = findUpstreamBranchPoint()
2159         if len(upstream) == 0:
2160             die("Cannot find upstream branchpoint for rebase")
2161
2162         # the branchpoint may be p4/foo~3, so strip off the parent
2163         upstream = re.sub("~[0-9]+$", "", upstream)
2164
2165         print "Rebasing the current branch onto %s" % upstream
2166         oldHead = read_pipe("git rev-parse HEAD").strip()
2167         system("git rebase %s" % upstream)
2168         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2169         return True
2170
2171 class P4Clone(P4Sync):
2172     def __init__(self):
2173         P4Sync.__init__(self)
2174         self.description = "Creates a new git repository and imports from Perforce into it"
2175         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2176         self.options += [
2177             optparse.make_option("--destination", dest="cloneDestination",
2178                                  action='store', default=None,
2179                                  help="where to leave result of the clone"),
2180             optparse.make_option("-/", dest="cloneExclude",
2181                                  action="append", type="string",
2182                                  help="exclude depot path"),
2183             optparse.make_option("--bare", dest="cloneBare",
2184                                  action="store_true", default=False),
2185         ]
2186         self.cloneDestination = None
2187         self.needsGit = False
2188         self.cloneBare = False
2189
2190     # This is required for the "append" cloneExclude action
2191     def ensure_value(self, attr, value):
2192         if not hasattr(self, attr) or getattr(self, attr) is None:
2193             setattr(self, attr, value)
2194         return getattr(self, attr)
2195
2196     def defaultDestination(self, args):
2197         ## TODO: use common prefix of args?
2198         depotPath = args[0]
2199         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2200         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2201         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2202         depotDir = re.sub(r"/$", "", depotDir)
2203         return os.path.split(depotDir)[1]
2204
2205     def run(self, args):
2206         if len(args) < 1:
2207             return False
2208
2209         if self.keepRepoPath and not self.cloneDestination:
2210             sys.stderr.write("Must specify destination for --keep-path\n")
2211             sys.exit(1)
2212
2213         depotPaths = args
2214
2215         if not self.cloneDestination and len(depotPaths) > 1:
2216             self.cloneDestination = depotPaths[-1]
2217             depotPaths = depotPaths[:-1]
2218
2219         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2220         for p in depotPaths:
2221             if not p.startswith("//"):
2222                 return False
2223
2224         if not self.cloneDestination:
2225             self.cloneDestination = self.defaultDestination(args)
2226
2227         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2228
2229         if not os.path.exists(self.cloneDestination):
2230             os.makedirs(self.cloneDestination)
2231         chdir(self.cloneDestination)
2232
2233         init_cmd = [ "git", "init" ]
2234         if self.cloneBare:
2235             init_cmd.append("--bare")
2236         subprocess.check_call(init_cmd)
2237
2238         if not P4Sync.run(self, depotPaths):
2239             return False
2240         if self.branch != "master":
2241             if self.importIntoRemotes:
2242                 masterbranch = "refs/remotes/p4/master"
2243             else:
2244                 masterbranch = "refs/heads/p4/master"
2245             if gitBranchExists(masterbranch):
2246                 system("git branch master %s" % masterbranch)
2247                 if not self.cloneBare:
2248                     system("git checkout -f")
2249             else:
2250                 print "Could not detect main branch. No checkout/master branch created."
2251
2252         return True
2253
2254 class P4Branches(Command):
2255     def __init__(self):
2256         Command.__init__(self)
2257         self.options = [ ]
2258         self.description = ("Shows the git branches that hold imports and their "
2259                             + "corresponding perforce depot paths")
2260         self.verbose = False
2261
2262     def run(self, args):
2263         if originP4BranchesExist():
2264             createOrUpdateBranchesFromOrigin()
2265
2266         cmdline = "git rev-parse --symbolic "
2267         cmdline += " --remotes"
2268
2269         for line in read_pipe_lines(cmdline):
2270             line = line.strip()
2271
2272             if not line.startswith('p4/') or line == "p4/HEAD":
2273                 continue
2274             branch = line
2275
2276             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2277             settings = extractSettingsGitLog(log)
2278
2279             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2280         return True
2281
2282 class HelpFormatter(optparse.IndentedHelpFormatter):
2283     def __init__(self):
2284         optparse.IndentedHelpFormatter.__init__(self)
2285
2286     def format_description(self, description):
2287         if description:
2288             return description + "\n"
2289         else:
2290             return ""
2291
2292 def printUsage(commands):
2293     print "usage: %s <command> [options]" % sys.argv[0]
2294     print ""
2295     print "valid commands: %s" % ", ".join(commands)
2296     print ""
2297     print "Try %s <command> --help for command specific help." % sys.argv[0]
2298     print ""
2299
2300 commands = {
2301     "debug" : P4Debug,
2302     "submit" : P4Submit,
2303     "commit" : P4Submit,
2304     "sync" : P4Sync,
2305     "rebase" : P4Rebase,
2306     "clone" : P4Clone,
2307     "rollback" : P4RollBack,
2308     "branches" : P4Branches
2309 }
2310
2311
2312 def main():
2313     if len(sys.argv[1:]) == 0:
2314         printUsage(commands.keys())
2315         sys.exit(2)
2316
2317     cmd = ""
2318     cmdName = sys.argv[1]
2319     try:
2320         klass = commands[cmdName]
2321         cmd = klass()
2322     except KeyError:
2323         print "unknown command %s" % cmdName
2324         print ""
2325         printUsage(commands.keys())
2326         sys.exit(2)
2327
2328     options = cmd.options
2329     cmd.gitdir = os.environ.get("GIT_DIR", None)
2330
2331     args = sys.argv[2:]
2332
2333     if len(options) > 0:
2334         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2335
2336         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2337                                        options,
2338                                        description = cmd.description,
2339                                        formatter = HelpFormatter())
2340
2341         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2342     global verbose
2343     verbose = cmd.verbose
2344     if cmd.needsGit:
2345         if cmd.gitdir == None:
2346             cmd.gitdir = os.path.abspath(".git")
2347             if not isValidGitDir(cmd.gitdir):
2348                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2349                 if os.path.exists(cmd.gitdir):
2350                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2351                     if len(cdup) > 0:
2352                         chdir(cdup);
2353
2354         if not isValidGitDir(cmd.gitdir):
2355             if isValidGitDir(cmd.gitdir + "/.git"):
2356                 cmd.gitdir += "/.git"
2357             else:
2358                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2359
2360         os.environ["GIT_DIR"] = cmd.gitdir
2361
2362     if not cmd.run(args):
2363         parser.print_help()
2364
2365
2366 if __name__ == '__main__':
2367     main()