contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19 def die(msg):
  20     if verbose:
  21         raise Exception(msg)
  22     else:
  23         sys.stderr.write(msg + "\n")
  24         sys.exit(1)
  25
  26 def write_pipe(c, str):
  27     if verbose:
  28         sys.stderr.write('Writing pipe: %s\n' % c)
  29
  30     pipe = os.popen(c, 'w')
  31     val = pipe.write(str)
  32     if pipe.close():
  33         die('Command failed: %s' % c)
  34
  35     return val
  36
  37 def read_pipe(c, ignore_error=False):
  38     if verbose:
  39         sys.stderr.write('Reading pipe: %s\n' % c)
  40
  41     pipe = os.popen(c, 'rb')
  42     val = pipe.read()
  43     if pipe.close() and not ignore_error:
  44         die('Command failed: %s' % c)
  45
  46     return val
  47
  48
  49 def read_pipe_lines(c):
  50     if verbose:
  51         sys.stderr.write('Reading pipe: %s\n' % c)
  52     ## todo: check return status
  53     pipe = os.popen(c, 'rb')
  54     val = pipe.readlines()
  55     if pipe.close():
  56         die('Command failed: %s' % c)
  57
  58     return val
  59
  60 def system(cmd):
  61     if verbose:
  62         sys.stderr.write("executing %s\n" % cmd)
  63     if os.system(cmd) != 0:
  64         die("command failed: %s" % cmd)
  65
  66 def isP4Exec(kind):
  67     """Determine if a Perforce 'kind' should have execute permission
  68
  69     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
  70     or x follows one of a few letters.  Otherwise, if there is an 'x' after
  71     a plus sign, it is also executable"""
  72     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
  73
  74 def setP4ExecBit(file, mode):
  75     # Reopens an already open file and changes the execute bit to match
  76     # the execute bit setting in the passed in mode.
  77
  78     p4Type = "+x"
  79
  80     if not isModeExec(mode):
  81         p4Type = getP4OpenedType(file)
  82         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
  83         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
  84         if p4Type[-1] == "+":
  85             p4Type = p4Type[0:-1]
  86
  87     system("p4 reopen -t %s %s" % (p4Type, file))
  88
  89 def getP4OpenedType(file):
  90     # Returns the perforce file type for the given file.
  91
  92     result = read_pipe("p4 opened %s" % file)
  93     match = re.match(".*\((.+)\)\r?$", result)
  94     if match:
  95         return match.group(1)
  96     else:
  97         die("Could not determine file type for %s (result: '%s')" % (file, result))
  98
  99 def diffTreePattern():
 100     # This is a simple generator for the diff tree regex pattern. This could be
 101     # a class variable if this and parseDiffTreeEntry were a part of a class.
 102     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 103     while True:
 104         yield pattern
 105
 106 def parseDiffTreeEntry(entry):
 107     """Parses a single diff tree entry into its component elements.
 108
 109     See git-diff-tree(1) manpage for details about the format of the diff
 110     output. This method returns a dictionary with the following elements:
 111
 112     src_mode - The mode of the source file
 113     dst_mode - The mode of the destination file
 114     src_sha1 - The sha1 for the source file
 115     dst_sha1 - The sha1 fr the destination file
 116     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 117     status_score - The score for the status (applicable for 'C' and 'R'
 118                    statuses). This is None if there is no score.
 119     src - The path for the source file.
 120     dst - The path for the destination file. This is only present for
 121           copy or renames. If it is not present, this is None.
 122
 123     If the pattern is not matched, None is returned."""
 124
 125     match = diffTreePattern().next().match(entry)
 126     if match:
 127         return {
 128             'src_mode': match.group(1),
 129             'dst_mode': match.group(2),
 130             'src_sha1': match.group(3),
 131             'dst_sha1': match.group(4),
 132             'status': match.group(5),
 133             'status_score': match.group(6),
 134             'src': match.group(7),
 135             'dst': match.group(10)
 136         }
 137     return None
 138
 139 def isModeExec(mode):
 140     # Returns True if the given git mode represents an executable file,
 141     # otherwise False.
 142     return mode[-3:] == "755"
 143
 144 def isModeExecChanged(src_mode, dst_mode):
 145     return isModeExec(src_mode) != isModeExec(dst_mode)
 146
 147 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 148     cmd = "p4 -G %s" % cmd
 149     if verbose:
 150         sys.stderr.write("Opening pipe: %s\n" % cmd)
 151
 152     # Use a temporary file to avoid deadlocks without
 153     # subprocess.communicate(), which would put another copy
 154     # of stdout into memory.
 155     stdin_file = None
 156     if stdin is not None:
 157         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 158         stdin_file.write(stdin)
 159         stdin_file.flush()
 160         stdin_file.seek(0)
 161
 162     p4 = subprocess.Popen(cmd, shell=True,
 163                           stdin=stdin_file,
 164                           stdout=subprocess.PIPE)
 165
 166     result = []
 167     try:
 168         while True:
 169             entry = marshal.load(p4.stdout)
 170             result.append(entry)
 171     except EOFError:
 172         pass
 173     exitCode = p4.wait()
 174     if exitCode != 0:
 175         entry = {}
 176         entry["p4ExitCode"] = exitCode
 177         result.append(entry)
 178
 179     return result
 180
 181 def p4Cmd(cmd):
 182     list = p4CmdList(cmd)
 183     result = {}
 184     for entry in list:
 185         result.update(entry)
 186     return result;
 187
 188 def p4Where(depotPath):
 189     if not depotPath.endswith("/"):
 190         depotPath += "/"
 191     output = p4Cmd("where %s..." % depotPath)
 192     if output["code"] == "error":
 193         return ""
 194     clientPath = ""
 195     if "path" in output:
 196         clientPath = output.get("path")
 197     elif "data" in output:
 198         data = output.get("data")
 199         lastSpace = data.rfind(" ")
 200         clientPath = data[lastSpace + 1:]
 201
 202     if clientPath.endswith("..."):
 203         clientPath = clientPath[:-3]
 204     return clientPath
 205
 206 def currentGitBranch():
 207     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 208
 209 def isValidGitDir(path):
 210     if (os.path.exists(path + "/HEAD")
 211         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 212         return True;
 213     return False
 214
 215 def parseRevision(ref):
 216     return read_pipe("git rev-parse %s" % ref).strip()
 217
 218 def extractLogMessageFromGitCommit(commit):
 219     logMessage = ""
 220
 221     ## fixme: title is first line of commit, not 1st paragraph.
 222     foundTitle = False
 223     for log in read_pipe_lines("git cat-file commit %s" % commit):
 224        if not foundTitle:
 225            if len(log) == 1:
 226                foundTitle = True
 227            continue
 228
 229        logMessage += log
 230     return logMessage
 231
 232 def extractSettingsGitLog(log):
 233     values = {}
 234     for line in log.split("\n"):
 235         line = line.strip()
 236         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 237         if not m:
 238             continue
 239
 240         assignments = m.group(1).split (':')
 241         for a in assignments:
 242             vals = a.split ('=')
 243             key = vals[0].strip()
 244             val = ('='.join (vals[1:])).strip()
 245             if val.endswith ('\"') and val.startswith('"'):
 246                 val = val[1:-1]
 247
 248             values[key] = val
 249
 250     paths = values.get("depot-paths")
 251     if not paths:
 252         paths = values.get("depot-path")
 253     if paths:
 254         values['depot-paths'] = paths.split(',')
 255     return values
 256
 257 def gitBranchExists(branch):
 258     proc = subprocess.Popen(["git", "rev-parse", branch],
 259                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 260     return proc.wait() == 0;
 261
 262 def gitConfig(key):
 263     return read_pipe("git config %s" % key, ignore_error=True).strip()
 264
 265 def p4BranchesInGit(branchesAreInRemotes = True):
 266     branches = {}
 267
 268     cmdline = "git rev-parse --symbolic "
 269     if branchesAreInRemotes:
 270         cmdline += " --remotes"
 271     else:
 272         cmdline += " --branches"
 273
 274     for line in read_pipe_lines(cmdline):
 275         line = line.strip()
 276
 277         ## only import to p4/
 278         if not line.startswith('p4/') or line == "p4/HEAD":
 279             continue
 280         branch = line
 281
 282         # strip off p4
 283         branch = re.sub ("^p4/", "", line)
 284
 285         branches[branch] = parseRevision(line)
 286     return branches
 287
 288 def findUpstreamBranchPoint(head = "HEAD"):
 289     branches = p4BranchesInGit()
 290     # map from depot-path to branch name
 291     branchByDepotPath = {}
 292     for branch in branches.keys():
 293         tip = branches[branch]
 294         log = extractLogMessageFromGitCommit(tip)
 295         settings = extractSettingsGitLog(log)
 296         if settings.has_key("depot-paths"):
 297             paths = ",".join(settings["depot-paths"])
 298             branchByDepotPath[paths] = "remotes/p4/" + branch
 299
 300     settings = None
 301     parent = 0
 302     while parent < 65535:
 303         commit = head + "~%s" % parent
 304         log = extractLogMessageFromGitCommit(commit)
 305         settings = extractSettingsGitLog(log)
 306         if settings.has_key("depot-paths"):
 307             paths = ",".join(settings["depot-paths"])
 308             if branchByDepotPath.has_key(paths):
 309                 return [branchByDepotPath[paths], settings]
 310
 311         parent = parent + 1
 312
 313     return ["", settings]
 314
 315 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 316     if not silent:
 317         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 318                % localRefPrefix)
 319
 320     originPrefix = "origin/p4/"
 321
 322     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 323         line = line.strip()
 324         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 325             continue
 326
 327         headName = line[len(originPrefix):]
 328         remoteHead = localRefPrefix + headName
 329         originHead = line
 330
 331         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 332         if (not original.has_key('depot-paths')
 333             or not original.has_key('change')):
 334             continue
 335
 336         update = False
 337         if not gitBranchExists(remoteHead):
 338             if verbose:
 339                 print "creating %s" % remoteHead
 340             update = True
 341         else:
 342             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 343             if settings.has_key('change') > 0:
 344                 if settings['depot-paths'] == original['depot-paths']:
 345                     originP4Change = int(original['change'])
 346                     p4Change = int(settings['change'])
 347                     if originP4Change > p4Change:
 348                         print ("%s (%s) is newer than %s (%s). "
 349                                "Updating p4 branch from origin."
 350                                % (originHead, originP4Change,
 351                                   remoteHead, p4Change))
 352                         update = True
 353                 else:
 354                     print ("Ignoring: %s was imported from %s while "
 355                            "%s was imported from %s"
 356                            % (originHead, ','.join(original['depot-paths']),
 357                               remoteHead, ','.join(settings['depot-paths'])))
 358
 359         if update:
 360             system("git update-ref %s %s" % (remoteHead, originHead))
 361
 362 def originP4BranchesExist():
 363         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 364
 365 def p4ChangesForPaths(depotPaths, changeRange):
 366     assert depotPaths
 367     output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, changeRange)
 368                                                         for p in depotPaths]))
 369
 370     changes = []
 371     for line in output:
 372         changeNum = line.split(" ")[1]
 373         changes.append(int(changeNum))
 374
 375     changes.sort()
 376     return changes
 377
 378 class Command:
 379     def __init__(self):
 380         self.usage = "usage: %prog [options]"
 381         self.needsGit = True
 382
 383 class P4Debug(Command):
 384     def __init__(self):
 385         Command.__init__(self)
 386         self.options = [
 387             optparse.make_option("--verbose", dest="verbose", action="store_true",
 388                                  default=False),
 389             ]
 390         self.description = "A tool to debug the output of p4 -G."
 391         self.needsGit = False
 392         self.verbose = False
 393
 394     def run(self, args):
 395         j = 0
 396         for output in p4CmdList(" ".join(args)):
 397             print 'Element: %d' % j
 398             j += 1
 399             print output
 400         return True
 401
 402 class P4RollBack(Command):
 403     def __init__(self):
 404         Command.__init__(self)
 405         self.options = [
 406             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 407             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 408         ]
 409         self.description = "A tool to debug the multi-branch import. Don't use :)"
 410         self.verbose = False
 411         self.rollbackLocalBranches = False
 412
 413     def run(self, args):
 414         if len(args) != 1:
 415             return False
 416         maxChange = int(args[0])
 417
 418         if "p4ExitCode" in p4Cmd("changes -m 1"):
 419             die("Problems executing p4");
 420
 421         if self.rollbackLocalBranches:
 422             refPrefix = "refs/heads/"
 423             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 424         else:
 425             refPrefix = "refs/remotes/"
 426             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 427
 428         for line in lines:
 429             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 430                 line = line.strip()
 431                 ref = refPrefix + line
 432                 log = extractLogMessageFromGitCommit(ref)
 433                 settings = extractSettingsGitLog(log)
 434
 435                 depotPaths = settings['depot-paths']
 436                 change = settings['change']
 437
 438                 changed = False
 439
 440                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 441                                                            for p in depotPaths]))) == 0:
 442                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 443                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 444                     continue
 445
 446                 while change and int(change) > maxChange:
 447                     changed = True
 448                     if self.verbose:
 449                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 450                     system("git update-ref %s \"%s^\"" % (ref, ref))
 451                     log = extractLogMessageFromGitCommit(ref)
 452                     settings =  extractSettingsGitLog(log)
 453
 454
 455                     depotPaths = settings['depot-paths']
 456                     change = settings['change']
 457
 458                 if changed:
 459                     print "%s rewound to %s" % (ref, change)
 460
 461         return True
 462
 463 class P4Submit(Command):
 464     def __init__(self):
 465         Command.__init__(self)
 466         self.options = [
 467                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 468                 optparse.make_option("--origin", dest="origin"),
 469                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 470         ]
 471         self.description = "Submit changes from git to the perforce depot."
 472         self.usage += " [name of git branch to submit into perforce depot]"
 473         self.interactive = True
 474         self.origin = ""
 475         self.detectRename = False
 476         self.verbose = False
 477         self.isWindows = (platform.system() == "Windows")
 478
 479     def check(self):
 480         if len(p4CmdList("opened ...")) > 0:
 481             die("You have files opened with perforce! Close them before starting the sync.")
 482
 483     # replaces everything between 'Description:' and the next P4 submit template field with the
 484     # commit message
 485     def prepareLogMessage(self, template, message):
 486         result = ""
 487
 488         inDescriptionSection = False
 489
 490         for line in template.split("\n"):
 491             if line.startswith("#"):
 492                 result += line + "\n"
 493                 continue
 494
 495             if inDescriptionSection:
 496                 if line.startswith("Files:"):
 497                     inDescriptionSection = False
 498                 else:
 499                     continue
 500             else:
 501                 if line.startswith("Description:"):
 502                     inDescriptionSection = True
 503                     line += "\n"
 504                     for messageLine in message.split("\n"):
 505                         line += "\t" + messageLine + "\n"
 506
 507             result += line + "\n"
 508
 509         return result
 510
 511     def prepareSubmitTemplate(self):
 512         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 513         template = ""
 514         inFilesSection = False
 515         for line in read_pipe_lines("p4 change -o"):
 516             if line.endswith("\r\n"):
 517                 line = line[:-2] + "\n"
 518             if inFilesSection:
 519                 if line.startswith("\t"):
 520                     # path starts and ends with a tab
 521                     path = line[1:]
 522                     lastTab = path.rfind("\t")
 523                     if lastTab != -1:
 524                         path = path[:lastTab]
 525                         if not path.startswith(self.depotPath):
 526                             continue
 527                 else:
 528                     inFilesSection = False
 529             else:
 530                 if line.startswith("Files:"):
 531                     inFilesSection = True
 532
 533             template += line
 534
 535         return template
 536
 537     def applyCommit(self, id):
 538         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 539         diffOpts = ("", "-M")[self.detectRename]
 540         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 541         filesToAdd = set()
 542         filesToDelete = set()
 543         editedFiles = set()
 544         filesToChangeExecBit = {}
 545         for line in diff:
 546             diff = parseDiffTreeEntry(line)
 547             modifier = diff['status']
 548             path = diff['src']
 549             if modifier == "M":
 550                 system("p4 edit \"%s\"" % path)
 551                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 552                     filesToChangeExecBit[path] = diff['dst_mode']
 553                 editedFiles.add(path)
 554             elif modifier == "A":
 555                 filesToAdd.add(path)
 556                 filesToChangeExecBit[path] = diff['dst_mode']
 557                 if path in filesToDelete:
 558                     filesToDelete.remove(path)
 559             elif modifier == "D":
 560                 filesToDelete.add(path)
 561                 if path in filesToAdd:
 562                     filesToAdd.remove(path)
 563             elif modifier == "R":
 564                 src, dest = diff['src'], diff['dst']
 565                 system("p4 integrate -Dt \"%s\" \"%s\"" % (src, dest))
 566                 system("p4 edit \"%s\"" % (dest))
 567                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 568                     filesToChangeExecBit[dest] = diff['dst_mode']
 569                 os.unlink(dest)
 570                 editedFiles.add(dest)
 571                 filesToDelete.add(src)
 572             else:
 573                 die("unknown modifier %s for %s" % (modifier, path))
 574
 575         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 576         patchcmd = diffcmd + " | git apply "
 577         tryPatchCmd = patchcmd + "--check -"
 578         applyPatchCmd = patchcmd + "--check --apply -"
 579
 580         if os.system(tryPatchCmd) != 0:
 581             print "Unfortunately applying the change failed!"
 582             print "What do you want to do?"
 583             response = "x"
 584             while response != "s" and response != "a" and response != "w":
 585                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 586                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 587             if response == "s":
 588                 print "Skipping! Good luck with the next patches..."
 589                 for f in editedFiles:
 590                     system("p4 revert \"%s\"" % f);
 591                 for f in filesToAdd:
 592                     system("rm %s" %f)
 593                 return
 594             elif response == "a":
 595                 os.system(applyPatchCmd)
 596                 if len(filesToAdd) > 0:
 597                     print "You may also want to call p4 add on the following files:"
 598                     print " ".join(filesToAdd)
 599                 if len(filesToDelete):
 600                     print "The following files should be scheduled for deletion with p4 delete:"
 601                     print " ".join(filesToDelete)
 602                 die("Please resolve and submit the conflict manually and "
 603                     + "continue afterwards with git-p4 submit --continue")
 604             elif response == "w":
 605                 system(diffcmd + " > patch.txt")
 606                 print "Patch saved to patch.txt in %s !" % self.clientPath
 607                 die("Please resolve and submit the conflict manually and "
 608                     "continue afterwards with git-p4 submit --continue")
 609
 610         system(applyPatchCmd)
 611
 612         for f in filesToAdd:
 613             system("p4 add \"%s\"" % f)
 614         for f in filesToDelete:
 615             system("p4 revert \"%s\"" % f)
 616             system("p4 delete \"%s\"" % f)
 617
 618         # Set/clear executable bits
 619         for f in filesToChangeExecBit.keys():
 620             mode = filesToChangeExecBit[f]
 621             setP4ExecBit(f, mode)
 622
 623         logMessage = extractLogMessageFromGitCommit(id)
 624         logMessage = logMessage.strip()
 625
 626         template = self.prepareSubmitTemplate()
 627
 628         if self.interactive:
 629             submitTemplate = self.prepareLogMessage(template, logMessage)
 630             if os.environ.has_key("P4DIFF"):
 631                 del(os.environ["P4DIFF"])
 632             diff = read_pipe("p4 diff -du ...")
 633
 634             newdiff = ""
 635             for newFile in filesToAdd:
 636                 newdiff += "==== new file ====\n"
 637                 newdiff += "--- /dev/null\n"
 638                 newdiff += "+++ %s\n" % newFile
 639                 f = open(newFile, "r")
 640                 for line in f.readlines():
 641                     newdiff += "+" + line
 642                 f.close()
 643
 644             separatorLine = "######## everything below this line is just the diff #######\n"
 645
 646             [handle, fileName] = tempfile.mkstemp()
 647             tmpFile = os.fdopen(handle, "w+")
 648             if self.isWindows:
 649                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 650                 separatorLine = separatorLine.replace("\n", "\r\n")
 651                 newdiff = newdiff.replace("\n", "\r\n")
 652             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 653             tmpFile.close()
 654             defaultEditor = "vi"
 655             if platform.system() == "Windows":
 656                 defaultEditor = "notepad"
 657             if os.environ.has_key("P4EDITOR"):
 658                 editor = os.environ.get("P4EDITOR")
 659             else:
 660                 editor = os.environ.get("EDITOR", defaultEditor);
 661             system(editor + " " + fileName)
 662             tmpFile = open(fileName, "rb")
 663             message = tmpFile.read()
 664             tmpFile.close()
 665             os.remove(fileName)
 666             submitTemplate = message[:message.index(separatorLine)]
 667             if self.isWindows:
 668                 submitTemplate = submitTemplate.replace("\r\n", "\n")
 669
 670             write_pipe("p4 submit -i", submitTemplate)
 671         else:
 672             fileName = "submit.txt"
 673             file = open(fileName, "w+")
 674             file.write(self.prepareLogMessage(template, logMessage))
 675             file.close()
 676             print ("Perforce submit template written as %s. "
 677                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 678                    % (fileName, fileName))
 679
 680     def run(self, args):
 681         if len(args) == 0:
 682             self.master = currentGitBranch()
 683             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 684                 die("Detecting current git branch failed!")
 685         elif len(args) == 1:
 686             self.master = args[0]
 687         else:
 688             return False
 689
 690         [upstream, settings] = findUpstreamBranchPoint()
 691         self.depotPath = settings['depot-paths'][0]
 692         if len(self.origin) == 0:
 693             self.origin = upstream
 694
 695         if self.verbose:
 696             print "Origin branch is " + self.origin
 697
 698         if len(self.depotPath) == 0:
 699             print "Internal error: cannot locate perforce depot path from existing branches"
 700             sys.exit(128)
 701
 702         self.clientPath = p4Where(self.depotPath)
 703
 704         if len(self.clientPath) == 0:
 705             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 706             sys.exit(128)
 707
 708         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 709         self.oldWorkingDirectory = os.getcwd()
 710
 711         os.chdir(self.clientPath)
 712         print "Syncronizing p4 checkout..."
 713         system("p4 sync ...")
 714
 715         self.check()
 716
 717         commits = []
 718         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 719             commits.append(line.strip())
 720         commits.reverse()
 721
 722         while len(commits) > 0:
 723             commit = commits[0]
 724             commits = commits[1:]
 725             self.applyCommit(commit)
 726             if not self.interactive:
 727                 break
 728
 729         if len(commits) == 0:
 730             print "All changes applied!"
 731             os.chdir(self.oldWorkingDirectory)
 732
 733             sync = P4Sync()
 734             sync.run([])
 735
 736             rebase = P4Rebase()
 737             rebase.rebase()
 738
 739         return True
 740
 741 class P4Sync(Command):
 742     def __init__(self):
 743         Command.__init__(self)
 744         self.options = [
 745                 optparse.make_option("--branch", dest="branch"),
 746                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 747                 optparse.make_option("--changesfile", dest="changesFile"),
 748                 optparse.make_option("--silent", dest="silent", action="store_true"),
 749                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 750                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 751                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 752                                      help="Import into refs/heads/ , not refs/remotes"),
 753                 optparse.make_option("--max-changes", dest="maxChanges"),
 754                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 755                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 756                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 757                                      help="Only sync files that are included in the Perforce Client Spec")
 758         ]
 759         self.description = """Imports from Perforce into a git repository.\n
 760     example:
 761     //depot/my/project/ -- to import the current head
 762     //depot/my/project/@all -- to import everything
 763     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 764
 765     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 766
 767         self.usage += " //depot/path[@revRange]"
 768         self.silent = False
 769         self.createdBranches = Set()
 770         self.committedChanges = Set()
 771         self.branch = ""
 772         self.detectBranches = False
 773         self.detectLabels = False
 774         self.changesFile = ""
 775         self.syncWithOrigin = True
 776         self.verbose = False
 777         self.importIntoRemotes = True
 778         self.maxChanges = ""
 779         self.isWindows = (platform.system() == "Windows")
 780         self.keepRepoPath = False
 781         self.depotPaths = None
 782         self.p4BranchesInGit = []
 783         self.cloneExclude = []
 784         self.useClientSpec = False
 785         self.clientSpecDirs = []
 786
 787         if gitConfig("git-p4.syncFromOrigin") == "false":
 788             self.syncWithOrigin = False
 789
 790     def extractFilesFromCommit(self, commit):
 791         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 792                              for path in self.cloneExclude]
 793         files = []
 794         fnum = 0
 795         while commit.has_key("depotFile%s" % fnum):
 796             path =  commit["depotFile%s" % fnum]
 797
 798             if [p for p in self.cloneExclude
 799                 if path.startswith (p)]:
 800                 found = False
 801             else:
 802                 found = [p for p in self.depotPaths
 803                          if path.startswith (p)]
 804             if not found:
 805                 fnum = fnum + 1
 806                 continue
 807
 808             file = {}
 809             file["path"] = path
 810             file["rev"] = commit["rev%s" % fnum]
 811             file["action"] = commit["action%s" % fnum]
 812             file["type"] = commit["type%s" % fnum]
 813             files.append(file)
 814             fnum = fnum + 1
 815         return files
 816
 817     def stripRepoPath(self, path, prefixes):
 818         if self.keepRepoPath:
 819             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 820
 821         for p in prefixes:
 822             if path.startswith(p):
 823                 path = path[len(p):]
 824
 825         return path
 826
 827     def splitFilesIntoBranches(self, commit):
 828         branches = {}
 829         fnum = 0
 830         while commit.has_key("depotFile%s" % fnum):
 831             path =  commit["depotFile%s" % fnum]
 832             found = [p for p in self.depotPaths
 833                      if path.startswith (p)]
 834             if not found:
 835                 fnum = fnum + 1
 836                 continue
 837
 838             file = {}
 839             file["path"] = path
 840             file["rev"] = commit["rev%s" % fnum]
 841             file["action"] = commit["action%s" % fnum]
 842             file["type"] = commit["type%s" % fnum]
 843             fnum = fnum + 1
 844
 845             relPath = self.stripRepoPath(path, self.depotPaths)
 846
 847             for branch in self.knownBranches.keys():
 848
 849                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 850                 if relPath.startswith(branch + "/"):
 851                     if branch not in branches:
 852                         branches[branch] = []
 853                     branches[branch].append(file)
 854                     break
 855
 856         return branches
 857
 858     ## Should move this out, doesn't use SELF.
 859     def readP4Files(self, files):
 860         filesForCommit = []
 861         filesToRead = []
 862
 863         for f in files:
 864             includeFile = True
 865             for val in self.clientSpecDirs:
 866                 if f['path'].startswith(val[0]):
 867                     if val[1] <= 0:
 868                         includeFile = False
 869                     break
 870
 871             if includeFile:
 872                 filesForCommit.append(f)
 873                 if f['action'] != 'delete':
 874                     filesToRead.append(f)
 875
 876         filedata = []
 877         if len(filesToRead) > 0:
 878             filedata = p4CmdList('-x - print',
 879                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 880                                                   for f in filesToRead]),
 881                                  stdin_mode='w+')
 882
 883             if "p4ExitCode" in filedata[0]:
 884                 die("Problems executing p4. Error: [%d]."
 885                     % (filedata[0]['p4ExitCode']));
 886
 887         j = 0;
 888         contents = {}
 889         while j < len(filedata):
 890             stat = filedata[j]
 891             j += 1
 892             text = [];
 893             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 894                 text.append(filedata[j]['data'])
 895                 j += 1
 896             text = ''.join(text)
 897
 898             if not stat.has_key('depotFile'):
 899                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 900                 continue
 901
 902             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 903                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 904             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 905                 text = re.sub(r'(?i)\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$',r'$\1$', text)
 906
 907             contents[stat['depotFile']] = text
 908
 909         for f in filesForCommit:
 910             path = f['path']
 911             if contents.has_key(path):
 912                 f['data'] = contents[path]
 913
 914         return filesForCommit
 915
 916     def commit(self, details, files, branch, branchPrefixes, parent = ""):
 917         epoch = details["time"]
 918         author = details["user"]
 919
 920         if self.verbose:
 921             print "commit into %s" % branch
 922
 923         # start with reading files; if that fails, we should not
 924         # create a commit.
 925         new_files = []
 926         for f in files:
 927             if [p for p in branchPrefixes if f['path'].startswith(p)]:
 928                 new_files.append (f)
 929             else:
 930                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
 931         files = self.readP4Files(new_files)
 932
 933         self.gitStream.write("commit %s\n" % branch)
 934 #        gitStream.write("mark :%s\n" % details["change"])
 935         self.committedChanges.add(int(details["change"]))
 936         committer = ""
 937         if author not in self.users:
 938             self.getUserMapFromPerforceServer()
 939         if author in self.users:
 940             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 941         else:
 942             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 943
 944         self.gitStream.write("committer %s\n" % committer)
 945
 946         self.gitStream.write("data <<EOT\n")
 947         self.gitStream.write(details["desc"])
 948         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
 949                              % (','.join (branchPrefixes), details["change"]))
 950         if len(details['options']) > 0:
 951             self.gitStream.write(": options = %s" % details['options'])
 952         self.gitStream.write("]\nEOT\n\n")
 953
 954         if len(parent) > 0:
 955             if self.verbose:
 956                 print "parent %s" % parent
 957             self.gitStream.write("from %s\n" % parent)
 958
 959         for file in files:
 960             if file["type"] == "apple":
 961                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
 962                 continue
 963
 964             relPath = self.stripRepoPath(file['path'], branchPrefixes)
 965             if file["action"] == "delete":
 966                 self.gitStream.write("D %s\n" % relPath)
 967             else:
 968                 data = file['data']
 969
 970                 mode = "644"
 971                 if isP4Exec(file["type"]):
 972                     mode = "755"
 973                 elif file["type"] == "symlink":
 974                     mode = "120000"
 975                     # p4 print on a symlink contains "target\n", so strip it off
 976                     data = data[:-1]
 977
 978                 if self.isWindows and file["type"].endswith("text"):
 979                     data = data.replace("\r\n", "\n")
 980
 981                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 982                 self.gitStream.write("data %s\n" % len(data))
 983                 self.gitStream.write(data)
 984                 self.gitStream.write("\n")
 985
 986         self.gitStream.write("\n")
 987
 988         change = int(details["change"])
 989
 990         if self.labels.has_key(change):
 991             label = self.labels[change]
 992             labelDetails = label[0]
 993             labelRevisions = label[1]
 994             if self.verbose:
 995                 print "Change %s is labelled %s" % (change, labelDetails)
 996
 997             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
 998                                                     for p in branchPrefixes]))
 999
1000             if len(files) == len(labelRevisions):
1001
1002                 cleanedFiles = {}
1003                 for info in files:
1004                     if info["action"] == "delete":
1005                         continue
1006                     cleanedFiles[info["depotFile"]] = info["rev"]
1007
1008                 if cleanedFiles == labelRevisions:
1009                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1010                     self.gitStream.write("from %s\n" % branch)
1011
1012                     owner = labelDetails["Owner"]
1013                     tagger = ""
1014                     if author in self.users:
1015                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1016                     else:
1017                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1018                     self.gitStream.write("tagger %s\n" % tagger)
1019                     self.gitStream.write("data <<EOT\n")
1020                     self.gitStream.write(labelDetails["Description"])
1021                     self.gitStream.write("EOT\n\n")
1022
1023                 else:
1024                     if not self.silent:
1025                         print ("Tag %s does not match with change %s: files do not match."
1026                                % (labelDetails["label"], change))
1027
1028             else:
1029                 if not self.silent:
1030                     print ("Tag %s does not match with change %s: file count is different."
1031                            % (labelDetails["label"], change))
1032
1033     def getUserCacheFilename(self):
1034         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1035         return home + "/.gitp4-usercache.txt"
1036
1037     def getUserMapFromPerforceServer(self):
1038         if self.userMapFromPerforceServer:
1039             return
1040         self.users = {}
1041
1042         for output in p4CmdList("users"):
1043             if not output.has_key("User"):
1044                 continue
1045             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1046
1047
1048         s = ''
1049         for (key, val) in self.users.items():
1050             s += "%s\t%s\n" % (key, val)
1051
1052         open(self.getUserCacheFilename(), "wb").write(s)
1053         self.userMapFromPerforceServer = True
1054
1055     def loadUserMapFromCache(self):
1056         self.users = {}
1057         self.userMapFromPerforceServer = False
1058         try:
1059             cache = open(self.getUserCacheFilename(), "rb")
1060             lines = cache.readlines()
1061             cache.close()
1062             for line in lines:
1063                 entry = line.strip().split("\t")
1064                 self.users[entry[0]] = entry[1]
1065         except IOError:
1066             self.getUserMapFromPerforceServer()
1067
1068     def getLabels(self):
1069         self.labels = {}
1070
1071         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1072         if len(l) > 0 and not self.silent:
1073             print "Finding files belonging to labels in %s" % `self.depotPaths`
1074
1075         for output in l:
1076             label = output["label"]
1077             revisions = {}
1078             newestChange = 0
1079             if self.verbose:
1080                 print "Querying files for label %s" % label
1081             for file in p4CmdList("files "
1082                                   +  ' '.join (["%s...@%s" % (p, label)
1083                                                 for p in self.depotPaths])):
1084                 revisions[file["depotFile"]] = file["rev"]
1085                 change = int(file["change"])
1086                 if change > newestChange:
1087                     newestChange = change
1088
1089             self.labels[newestChange] = [output, revisions]
1090
1091         if self.verbose:
1092             print "Label changes: %s" % self.labels.keys()
1093
1094     def guessProjectName(self):
1095         for p in self.depotPaths:
1096             if p.endswith("/"):
1097                 p = p[:-1]
1098             p = p[p.strip().rfind("/") + 1:]
1099             if not p.endswith("/"):
1100                p += "/"
1101             return p
1102
1103     def getBranchMapping(self):
1104         lostAndFoundBranches = set()
1105
1106         for info in p4CmdList("branches"):
1107             details = p4Cmd("branch -o %s" % info["branch"])
1108             viewIdx = 0
1109             while details.has_key("View%s" % viewIdx):
1110                 paths = details["View%s" % viewIdx].split(" ")
1111                 viewIdx = viewIdx + 1
1112                 # require standard //depot/foo/... //depot/bar/... mapping
1113                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1114                     continue
1115                 source = paths[0]
1116                 destination = paths[1]
1117                 ## HACK
1118                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1119                     source = source[len(self.depotPaths[0]):-4]
1120                     destination = destination[len(self.depotPaths[0]):-4]
1121
1122                     if destination in self.knownBranches:
1123                         if not self.silent:
1124                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1125                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1126                         continue
1127
1128                     self.knownBranches[destination] = source
1129
1130                     lostAndFoundBranches.discard(destination)
1131
1132                     if source not in self.knownBranches:
1133                         lostAndFoundBranches.add(source)
1134
1135
1136         for branch in lostAndFoundBranches:
1137             self.knownBranches[branch] = branch
1138
1139     def getBranchMappingFromGitBranches(self):
1140         branches = p4BranchesInGit(self.importIntoRemotes)
1141         for branch in branches.keys():
1142             if branch == "master":
1143                 branch = "main"
1144             else:
1145                 branch = branch[len(self.projectName):]
1146             self.knownBranches[branch] = branch
1147
1148     def listExistingP4GitBranches(self):
1149         # branches holds mapping from name to commit
1150         branches = p4BranchesInGit(self.importIntoRemotes)
1151         self.p4BranchesInGit = branches.keys()
1152         for branch in branches.keys():
1153             self.initialParents[self.refPrefix + branch] = branches[branch]
1154
1155     def updateOptionDict(self, d):
1156         option_keys = {}
1157         if self.keepRepoPath:
1158             option_keys['keepRepoPath'] = 1
1159
1160         d["options"] = ' '.join(sorted(option_keys.keys()))
1161
1162     def readOptions(self, d):
1163         self.keepRepoPath = (d.has_key('options')
1164                              and ('keepRepoPath' in d['options']))
1165
1166     def gitRefForBranch(self, branch):
1167         if branch == "main":
1168             return self.refPrefix + "master"
1169
1170         if len(branch) <= 0:
1171             return branch
1172
1173         return self.refPrefix + self.projectName + branch
1174
1175     def gitCommitByP4Change(self, ref, change):
1176         if self.verbose:
1177             print "looking in ref " + ref + " for change %s using bisect..." % change
1178
1179         earliestCommit = ""
1180         latestCommit = parseRevision(ref)
1181
1182         while True:
1183             if self.verbose:
1184                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1185             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1186             if len(next) == 0:
1187                 if self.verbose:
1188                     print "argh"
1189                 return ""
1190             log = extractLogMessageFromGitCommit(next)
1191             settings = extractSettingsGitLog(log)
1192             currentChange = int(settings['change'])
1193             if self.verbose:
1194                 print "current change %s" % currentChange
1195
1196             if currentChange == change:
1197                 if self.verbose:
1198                     print "found %s" % next
1199                 return next
1200
1201             if currentChange < change:
1202                 earliestCommit = "^%s" % next
1203             else:
1204                 latestCommit = "%s" % next
1205
1206         return ""
1207
1208     def importNewBranch(self, branch, maxChange):
1209         # make fast-import flush all changes to disk and update the refs using the checkpoint
1210         # command so that we can try to find the branch parent in the git history
1211         self.gitStream.write("checkpoint\n\n");
1212         self.gitStream.flush();
1213         branchPrefix = self.depotPaths[0] + branch + "/"
1214         range = "@1,%s" % maxChange
1215         #print "prefix" + branchPrefix
1216         changes = p4ChangesForPaths([branchPrefix], range)
1217         if len(changes) <= 0:
1218             return False
1219         firstChange = changes[0]
1220         #print "first change in branch: %s" % firstChange
1221         sourceBranch = self.knownBranches[branch]
1222         sourceDepotPath = self.depotPaths[0] + sourceBranch
1223         sourceRef = self.gitRefForBranch(sourceBranch)
1224         #print "source " + sourceBranch
1225
1226         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1227         #print "branch parent: %s" % branchParentChange
1228         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1229         if len(gitParent) > 0:
1230             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1231             #print "parent git commit: %s" % gitParent
1232
1233         self.importChanges(changes)
1234         return True
1235
1236     def importChanges(self, changes):
1237         cnt = 1
1238         for change in changes:
1239             description = p4Cmd("describe %s" % change)
1240             self.updateOptionDict(description)
1241
1242             if not self.silent:
1243                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1244                 sys.stdout.flush()
1245             cnt = cnt + 1
1246
1247             try:
1248                 if self.detectBranches:
1249                     branches = self.splitFilesIntoBranches(description)
1250                     for branch in branches.keys():
1251                         ## HACK  --hwn
1252                         branchPrefix = self.depotPaths[0] + branch + "/"
1253
1254                         parent = ""
1255
1256                         filesForCommit = branches[branch]
1257
1258                         if self.verbose:
1259                             print "branch is %s" % branch
1260
1261                         self.updatedBranches.add(branch)
1262
1263                         if branch not in self.createdBranches:
1264                             self.createdBranches.add(branch)
1265                             parent = self.knownBranches[branch]
1266                             if parent == branch:
1267                                 parent = ""
1268                             else:
1269                                 fullBranch = self.projectName + branch
1270                                 if fullBranch not in self.p4BranchesInGit:
1271                                     if not self.silent:
1272                                         print("\n    Importing new branch %s" % fullBranch);
1273                                     if self.importNewBranch(branch, change - 1):
1274                                         parent = ""
1275                                         self.p4BranchesInGit.append(fullBranch)
1276                                     if not self.silent:
1277                                         print("\n    Resuming with change %s" % change);
1278
1279                                 if self.verbose:
1280                                     print "parent determined through known branches: %s" % parent
1281
1282                         branch = self.gitRefForBranch(branch)
1283                         parent = self.gitRefForBranch(parent)
1284
1285                         if self.verbose:
1286                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1287
1288                         if len(parent) == 0 and branch in self.initialParents:
1289                             parent = self.initialParents[branch]
1290                             del self.initialParents[branch]
1291
1292                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1293                 else:
1294                     files = self.extractFilesFromCommit(description)
1295                     self.commit(description, files, self.branch, self.depotPaths,
1296                                 self.initialParent)
1297                     self.initialParent = ""
1298             except IOError:
1299                 print self.gitError.read()
1300                 sys.exit(1)
1301
1302     def importHeadRevision(self, revision):
1303         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1304
1305         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1306         details["desc"] = ("Initial import of %s from the state at revision %s"
1307                            % (' '.join(self.depotPaths), revision))
1308         details["change"] = revision
1309         newestRevision = 0
1310
1311         fileCnt = 0
1312         for info in p4CmdList("files "
1313                               +  ' '.join(["%s...%s"
1314                                            % (p, revision)
1315                                            for p in self.depotPaths])):
1316
1317             if info['code'] == 'error':
1318                 sys.stderr.write("p4 returned an error: %s\n"
1319                                  % info['data'])
1320                 sys.exit(1)
1321
1322
1323             change = int(info["change"])
1324             if change > newestRevision:
1325                 newestRevision = change
1326
1327             if info["action"] == "delete":
1328                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1329                 #fileCnt = fileCnt + 1
1330                 continue
1331
1332             for prop in ["depotFile", "rev", "action", "type" ]:
1333                 details["%s%s" % (prop, fileCnt)] = info[prop]
1334
1335             fileCnt = fileCnt + 1
1336
1337         details["change"] = newestRevision
1338         self.updateOptionDict(details)
1339         try:
1340             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1341         except IOError:
1342             print "IO error with git fast-import. Is your git version recent enough?"
1343             print self.gitError.read()
1344
1345
1346     def getClientSpec(self):
1347         specList = p4CmdList( "client -o" )
1348         temp = {}
1349         for entry in specList:
1350             for k,v in entry.iteritems():
1351                 if k.startswith("View"):
1352                     if v.startswith('"'):
1353                         start = 1
1354                     else:
1355                         start = 0
1356                     index = v.find("...")
1357                     v = v[start:index]
1358                     if v.startswith("-"):
1359                         v = v[1:]
1360                         temp[v] = -len(v)
1361                     else:
1362                         temp[v] = len(v)
1363         self.clientSpecDirs = temp.items()
1364         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1365
1366     def run(self, args):
1367         self.depotPaths = []
1368         self.changeRange = ""
1369         self.initialParent = ""
1370         self.previousDepotPaths = []
1371
1372         # map from branch depot path to parent branch
1373         self.knownBranches = {}
1374         self.initialParents = {}
1375         self.hasOrigin = originP4BranchesExist()
1376         if not self.syncWithOrigin:
1377             self.hasOrigin = False
1378
1379         if self.importIntoRemotes:
1380             self.refPrefix = "refs/remotes/p4/"
1381         else:
1382             self.refPrefix = "refs/heads/p4/"
1383
1384         if self.syncWithOrigin and self.hasOrigin:
1385             if not self.silent:
1386                 print "Syncing with origin first by calling git fetch origin"
1387             system("git fetch origin")
1388
1389         if len(self.branch) == 0:
1390             self.branch = self.refPrefix + "master"
1391             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1392                 system("git update-ref %s refs/heads/p4" % self.branch)
1393                 system("git branch -D p4");
1394             # create it /after/ importing, when master exists
1395             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1396                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1397
1398         if self.useClientSpec or gitConfig("p4.useclientspec") == "true":
1399             self.getClientSpec()
1400
1401         # TODO: should always look at previous commits,
1402         # merge with previous imports, if possible.
1403         if args == []:
1404             if self.hasOrigin:
1405                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1406             self.listExistingP4GitBranches()
1407
1408             if len(self.p4BranchesInGit) > 1:
1409                 if not self.silent:
1410                     print "Importing from/into multiple branches"
1411                 self.detectBranches = True
1412
1413             if self.verbose:
1414                 print "branches: %s" % self.p4BranchesInGit
1415
1416             p4Change = 0
1417             for branch in self.p4BranchesInGit:
1418                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1419
1420                 settings = extractSettingsGitLog(logMsg)
1421
1422                 self.readOptions(settings)
1423                 if (settings.has_key('depot-paths')
1424                     and settings.has_key ('change')):
1425                     change = int(settings['change']) + 1
1426                     p4Change = max(p4Change, change)
1427
1428                     depotPaths = sorted(settings['depot-paths'])
1429                     if self.previousDepotPaths == []:
1430                         self.previousDepotPaths = depotPaths
1431                     else:
1432                         paths = []
1433                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1434                             for i in range(0, min(len(cur), len(prev))):
1435                                 if cur[i] <> prev[i]:
1436                                     i = i - 1
1437                                     break
1438
1439                             paths.append (cur[:i + 1])
1440
1441                         self.previousDepotPaths = paths
1442
1443             if p4Change > 0:
1444                 self.depotPaths = sorted(self.previousDepotPaths)
1445                 self.changeRange = "@%s,#head" % p4Change
1446                 if not self.detectBranches:
1447                     self.initialParent = parseRevision(self.branch)
1448                 if not self.silent and not self.detectBranches:
1449                     print "Performing incremental import into %s git branch" % self.branch
1450
1451         if not self.branch.startswith("refs/"):
1452             self.branch = "refs/heads/" + self.branch
1453
1454         if len(args) == 0 and self.depotPaths:
1455             if not self.silent:
1456                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1457         else:
1458             if self.depotPaths and self.depotPaths != args:
1459                 print ("previous import used depot path %s and now %s was specified. "
1460                        "This doesn't work!" % (' '.join (self.depotPaths),
1461                                                ' '.join (args)))
1462                 sys.exit(1)
1463
1464             self.depotPaths = sorted(args)
1465
1466         revision = ""
1467         self.users = {}
1468
1469         newPaths = []
1470         for p in self.depotPaths:
1471             if p.find("@") != -1:
1472                 atIdx = p.index("@")
1473                 self.changeRange = p[atIdx:]
1474                 if self.changeRange == "@all":
1475                     self.changeRange = ""
1476                 elif ',' not in self.changeRange:
1477                     revision = self.changeRange
1478                     self.changeRange = ""
1479                 p = p[:atIdx]
1480             elif p.find("#") != -1:
1481                 hashIdx = p.index("#")
1482                 revision = p[hashIdx:]
1483                 p = p[:hashIdx]
1484             elif self.previousDepotPaths == []:
1485                 revision = "#head"
1486
1487             p = re.sub ("\.\.\.$", "", p)
1488             if not p.endswith("/"):
1489                 p += "/"
1490
1491             newPaths.append(p)
1492
1493         self.depotPaths = newPaths
1494
1495
1496         self.loadUserMapFromCache()
1497         self.labels = {}
1498         if self.detectLabels:
1499             self.getLabels();
1500
1501         if self.detectBranches:
1502             ## FIXME - what's a P4 projectName ?
1503             self.projectName = self.guessProjectName()
1504
1505             if self.hasOrigin:
1506                 self.getBranchMappingFromGitBranches()
1507             else:
1508                 self.getBranchMapping()
1509             if self.verbose:
1510                 print "p4-git branches: %s" % self.p4BranchesInGit
1511                 print "initial parents: %s" % self.initialParents
1512             for b in self.p4BranchesInGit:
1513                 if b != "master":
1514
1515                     ## FIXME
1516                     b = b[len(self.projectName):]
1517                 self.createdBranches.add(b)
1518
1519         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1520
1521         importProcess = subprocess.Popen(["git", "fast-import"],
1522                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1523                                          stderr=subprocess.PIPE);
1524         self.gitOutput = importProcess.stdout
1525         self.gitStream = importProcess.stdin
1526         self.gitError = importProcess.stderr
1527
1528         if revision:
1529             self.importHeadRevision(revision)
1530         else:
1531             changes = []
1532
1533             if len(self.changesFile) > 0:
1534                 output = open(self.changesFile).readlines()
1535                 changeSet = Set()
1536                 for line in output:
1537                     changeSet.add(int(line))
1538
1539                 for change in changeSet:
1540                     changes.append(change)
1541
1542                 changes.sort()
1543             else:
1544                 if self.verbose:
1545                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1546                                                               self.changeRange)
1547                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1548
1549                 if len(self.maxChanges) > 0:
1550                     changes = changes[:min(int(self.maxChanges), len(changes))]
1551
1552             if len(changes) == 0:
1553                 if not self.silent:
1554                     print "No changes to import!"
1555                 return True
1556
1557             if not self.silent and not self.detectBranches:
1558                 print "Import destination: %s" % self.branch
1559
1560             self.updatedBranches = set()
1561
1562             self.importChanges(changes)
1563
1564             if not self.silent:
1565                 print ""
1566                 if len(self.updatedBranches) > 0:
1567                     sys.stdout.write("Updated branches: ")
1568                     for b in self.updatedBranches:
1569                         sys.stdout.write("%s " % b)
1570                     sys.stdout.write("\n")
1571
1572         self.gitStream.close()
1573         if importProcess.wait() != 0:
1574             die("fast-import failed: %s" % self.gitError.read())
1575         self.gitOutput.close()
1576         self.gitError.close()
1577
1578         return True
1579
1580 class P4Rebase(Command):
1581     def __init__(self):
1582         Command.__init__(self)
1583         self.options = [ ]
1584         self.description = ("Fetches the latest revision from perforce and "
1585                             + "rebases the current work (branch) against it")
1586         self.verbose = False
1587
1588     def run(self, args):
1589         sync = P4Sync()
1590         sync.run([])
1591
1592         return self.rebase()
1593
1594     def rebase(self):
1595         if os.system("git update-index --refresh") != 0:
1596             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1597         if len(read_pipe("git diff-index HEAD --")) > 0:
1598             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1599
1600         [upstream, settings] = findUpstreamBranchPoint()
1601         if len(upstream) == 0:
1602             die("Cannot find upstream branchpoint for rebase")
1603
1604         # the branchpoint may be p4/foo~3, so strip off the parent
1605         upstream = re.sub("~[0-9]+$", "", upstream)
1606
1607         print "Rebasing the current branch onto %s" % upstream
1608         oldHead = read_pipe("git rev-parse HEAD").strip()
1609         system("git rebase %s" % upstream)
1610         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1611         return True
1612
1613 class P4Clone(P4Sync):
1614     def __init__(self):
1615         P4Sync.__init__(self)
1616         self.description = "Creates a new git repository and imports from Perforce into it"
1617         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1618         self.options += [
1619             optparse.make_option("--destination", dest="cloneDestination",
1620                                  action='store', default=None,
1621                                  help="where to leave result of the clone"),
1622             optparse.make_option("-/", dest="cloneExclude",
1623                                  action="append", type="string",
1624                                  help="exclude depot path")
1625         ]
1626         self.cloneDestination = None
1627         self.needsGit = False
1628
1629     # This is required for the "append" cloneExclude action
1630     def ensure_value(self, attr, value):
1631         if not hasattr(self, attr) or getattr(self, attr) is None:
1632             setattr(self, attr, value)
1633         return getattr(self, attr)
1634
1635     def defaultDestination(self, args):
1636         ## TODO: use common prefix of args?
1637         depotPath = args[0]
1638         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1639         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1640         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1641         depotDir = re.sub(r"/$", "", depotDir)
1642         return os.path.split(depotDir)[1]
1643
1644     def run(self, args):
1645         if len(args) < 1:
1646             return False
1647
1648         if self.keepRepoPath and not self.cloneDestination:
1649             sys.stderr.write("Must specify destination for --keep-path\n")
1650             sys.exit(1)
1651
1652         depotPaths = args
1653
1654         if not self.cloneDestination and len(depotPaths) > 1:
1655             self.cloneDestination = depotPaths[-1]
1656             depotPaths = depotPaths[:-1]
1657
1658         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1659         for p in depotPaths:
1660             if not p.startswith("//"):
1661                 return False
1662
1663         if not self.cloneDestination:
1664             self.cloneDestination = self.defaultDestination(args)
1665
1666         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1667         if not os.path.exists(self.cloneDestination):
1668             os.makedirs(self.cloneDestination)
1669         os.chdir(self.cloneDestination)
1670         system("git init")
1671         self.gitdir = os.getcwd() + "/.git"
1672         if not P4Sync.run(self, depotPaths):
1673             return False
1674         if self.branch != "master":
1675             if gitBranchExists("refs/remotes/p4/master"):
1676                 system("git branch master refs/remotes/p4/master")
1677                 system("git checkout -f")
1678             else:
1679                 print "Could not detect main branch. No checkout/master branch created."
1680
1681         return True
1682
1683 class P4Branches(Command):
1684     def __init__(self):
1685         Command.__init__(self)
1686         self.options = [ ]
1687         self.description = ("Shows the git branches that hold imports and their "
1688                             + "corresponding perforce depot paths")
1689         self.verbose = False
1690
1691     def run(self, args):
1692         if originP4BranchesExist():
1693             createOrUpdateBranchesFromOrigin()
1694
1695         cmdline = "git rev-parse --symbolic "
1696         cmdline += " --remotes"
1697
1698         for line in read_pipe_lines(cmdline):
1699             line = line.strip()
1700
1701             if not line.startswith('p4/') or line == "p4/HEAD":
1702                 continue
1703             branch = line
1704
1705             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1706             settings = extractSettingsGitLog(log)
1707
1708             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1709         return True
1710
1711 class HelpFormatter(optparse.IndentedHelpFormatter):
1712     def __init__(self):
1713         optparse.IndentedHelpFormatter.__init__(self)
1714
1715     def format_description(self, description):
1716         if description:
1717             return description + "\n"
1718         else:
1719             return ""
1720
1721 def printUsage(commands):
1722     print "usage: %s <command> [options]" % sys.argv[0]
1723     print ""
1724     print "valid commands: %s" % ", ".join(commands)
1725     print ""
1726     print "Try %s <command> --help for command specific help." % sys.argv[0]
1727     print ""
1728
1729 commands = {
1730     "debug" : P4Debug,
1731     "submit" : P4Submit,
1732     "commit" : P4Submit,
1733     "sync" : P4Sync,
1734     "rebase" : P4Rebase,
1735     "clone" : P4Clone,
1736     "rollback" : P4RollBack,
1737     "branches" : P4Branches
1738 }
1739
1740
1741 def main():
1742     if len(sys.argv[1:]) == 0:
1743         printUsage(commands.keys())
1744         sys.exit(2)
1745
1746     cmd = ""
1747     cmdName = sys.argv[1]
1748     try:
1749         klass = commands[cmdName]
1750         cmd = klass()
1751     except KeyError:
1752         print "unknown command %s" % cmdName
1753         print ""
1754         printUsage(commands.keys())
1755         sys.exit(2)
1756
1757     options = cmd.options
1758     cmd.gitdir = os.environ.get("GIT_DIR", None)
1759
1760     args = sys.argv[2:]
1761
1762     if len(options) > 0:
1763         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1764
1765         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1766                                        options,
1767                                        description = cmd.description,
1768                                        formatter = HelpFormatter())
1769
1770         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1771     global verbose
1772     verbose = cmd.verbose
1773     if cmd.needsGit:
1774         if cmd.gitdir == None:
1775             cmd.gitdir = os.path.abspath(".git")
1776             if not isValidGitDir(cmd.gitdir):
1777                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1778                 if os.path.exists(cmd.gitdir):
1779                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1780                     if len(cdup) > 0:
1781                         os.chdir(cdup);
1782
1783         if not isValidGitDir(cmd.gitdir):
1784             if isValidGitDir(cmd.gitdir + "/.git"):
1785                 cmd.gitdir += "/.git"
1786             else:
1787                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1788
1789         os.environ["GIT_DIR"] = cmd.gitdir
1790
1791     if not cmd.run(args):
1792         parser.print_help()
1793
1794
1795 if __name__ == '__main__':
1796     main()