contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, subprocess, shelve
  12 import tempfile, getopt, sha, os.path, time, platform
  13 import re
  14
  15 from sets import Set;
  16
  17 verbose = False
  18
  19
  20 def p4_build_cmd(cmd):
  21     """Build a suitable p4 command line.
  22
  23     This consolidates building and returning a p4 command line into one
  24     location. It means that hooking into the environment, or other configuration
  25     can be done more easily.
  26     """
  27     real_cmd = "%s " % "p4"
  28
  29     user = gitConfig("git-p4.user")
  30     if len(user) > 0:
  31         real_cmd += "-u %s " % user
  32
  33     password = gitConfig("git-p4.password")
  34     if len(password) > 0:
  35         real_cmd += "-P %s " % password
  36
  37     port = gitConfig("git-p4.port")
  38     if len(port) > 0:
  39         real_cmd += "-p %s " % port
  40
  41     host = gitConfig("git-p4.host")
  42     if len(host) > 0:
  43         real_cmd += "-h %s " % host
  44
  45     client = gitConfig("git-p4.client")
  46     if len(client) > 0:
  47         real_cmd += "-c %s " % client
  48
  49     real_cmd += "%s" % (cmd)
  50     if verbose:
  51         print real_cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     if os.name == 'nt':
  56         os.environ['PWD']=dir
  57     os.chdir(dir)
  58
  59 def die(msg):
  60     if verbose:
  61         raise Exception(msg)
  62     else:
  63         sys.stderr.write(msg + "\n")
  64         sys.exit(1)
  65
  66 def write_pipe(c, str):
  67     if verbose:
  68         sys.stderr.write('Writing pipe: %s\n' % c)
  69
  70     pipe = os.popen(c, 'w')
  71     val = pipe.write(str)
  72     if pipe.close():
  73         die('Command failed: %s' % c)
  74
  75     return val
  76
  77 def p4_write_pipe(c, str):
  78     real_cmd = p4_build_cmd(c)
  79     return write_pipe(real_cmd, str)
  80
  81 def read_pipe(c, ignore_error=False):
  82     if verbose:
  83         sys.stderr.write('Reading pipe: %s\n' % c)
  84
  85     pipe = os.popen(c, 'rb')
  86     val = pipe.read()
  87     if pipe.close() and not ignore_error:
  88         die('Command failed: %s' % c)
  89
  90     return val
  91
  92 def p4_read_pipe(c, ignore_error=False):
  93     real_cmd = p4_build_cmd(c)
  94     return read_pipe(real_cmd, ignore_error)
  95
  96 def read_pipe_lines(c):
  97     if verbose:
  98         sys.stderr.write('Reading pipe: %s\n' % c)
  99     ## todo: check return status
 100     pipe = os.popen(c, 'rb')
 101     val = pipe.readlines()
 102     if pipe.close():
 103         die('Command failed: %s' % c)
 104
 105     return val
 106
 107 def p4_read_pipe_lines(c):
 108     """Specifically invoke p4 on the command supplied. """
 109     real_cmd = p4_build_cmd(c)
 110     return read_pipe_lines(real_cmd)
 111
 112 def system(cmd):
 113     if verbose:
 114         sys.stderr.write("executing %s\n" % cmd)
 115     if os.system(cmd) != 0:
 116         die("command failed: %s" % cmd)
 117
 118 def p4_system(cmd):
 119     """Specifically invoke p4 as the system command. """
 120     real_cmd = p4_build_cmd(cmd)
 121     return system(real_cmd)
 122
 123 def isP4Exec(kind):
 124     """Determine if a Perforce 'kind' should have execute permission
 125
 126     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 127     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 128     a plus sign, it is also executable"""
 129     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 130
 131 def setP4ExecBit(file, mode):
 132     # Reopens an already open file and changes the execute bit to match
 133     # the execute bit setting in the passed in mode.
 134
 135     p4Type = "+x"
 136
 137     if not isModeExec(mode):
 138         p4Type = getP4OpenedType(file)
 139         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 140         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 141         if p4Type[-1] == "+":
 142             p4Type = p4Type[0:-1]
 143
 144     p4_system("reopen -t %s %s" % (p4Type, file))
 145
 146 def getP4OpenedType(file):
 147     # Returns the perforce file type for the given file.
 148
 149     result = p4_read_pipe("opened %s" % file)
 150     match = re.match(".*\((.+)\)\r?$", result)
 151     if match:
 152         return match.group(1)
 153     else:
 154         die("Could not determine file type for %s (result: '%s')" % (file, result))
 155
 156 def diffTreePattern():
 157     # This is a simple generator for the diff tree regex pattern. This could be
 158     # a class variable if this and parseDiffTreeEntry were a part of a class.
 159     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 160     while True:
 161         yield pattern
 162
 163 def parseDiffTreeEntry(entry):
 164     """Parses a single diff tree entry into its component elements.
 165
 166     See git-diff-tree(1) manpage for details about the format of the diff
 167     output. This method returns a dictionary with the following elements:
 168
 169     src_mode - The mode of the source file
 170     dst_mode - The mode of the destination file
 171     src_sha1 - The sha1 for the source file
 172     dst_sha1 - The sha1 fr the destination file
 173     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 174     status_score - The score for the status (applicable for 'C' and 'R'
 175                    statuses). This is None if there is no score.
 176     src - The path for the source file.
 177     dst - The path for the destination file. This is only present for
 178           copy or renames. If it is not present, this is None.
 179
 180     If the pattern is not matched, None is returned."""
 181
 182     match = diffTreePattern().next().match(entry)
 183     if match:
 184         return {
 185             'src_mode': match.group(1),
 186             'dst_mode': match.group(2),
 187             'src_sha1': match.group(3),
 188             'dst_sha1': match.group(4),
 189             'status': match.group(5),
 190             'status_score': match.group(6),
 191             'src': match.group(7),
 192             'dst': match.group(10)
 193         }
 194     return None
 195
 196 def isModeExec(mode):
 197     # Returns True if the given git mode represents an executable file,
 198     # otherwise False.
 199     return mode[-3:] == "755"
 200
 201 def isModeExecChanged(src_mode, dst_mode):
 202     return isModeExec(src_mode) != isModeExec(dst_mode)
 203
 204 def p4CmdList(cmd, stdin=None, stdin_mode='w+b'):
 205     cmd = p4_build_cmd("-G %s" % (cmd))
 206     if verbose:
 207         sys.stderr.write("Opening pipe: %s\n" % cmd)
 208
 209     # Use a temporary file to avoid deadlocks without
 210     # subprocess.communicate(), which would put another copy
 211     # of stdout into memory.
 212     stdin_file = None
 213     if stdin is not None:
 214         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 215         stdin_file.write(stdin)
 216         stdin_file.flush()
 217         stdin_file.seek(0)
 218
 219     p4 = subprocess.Popen(cmd, shell=True,
 220                           stdin=stdin_file,
 221                           stdout=subprocess.PIPE)
 222
 223     result = []
 224     try:
 225         while True:
 226             entry = marshal.load(p4.stdout)
 227             result.append(entry)
 228     except EOFError:
 229         pass
 230     exitCode = p4.wait()
 231     if exitCode != 0:
 232         entry = {}
 233         entry["p4ExitCode"] = exitCode
 234         result.append(entry)
 235
 236     return result
 237
 238 def p4Cmd(cmd):
 239     list = p4CmdList(cmd)
 240     result = {}
 241     for entry in list:
 242         result.update(entry)
 243     return result;
 244
 245 def p4Where(depotPath):
 246     if not depotPath.endswith("/"):
 247         depotPath += "/"
 248     depotPath = depotPath + "..."
 249     outputList = p4CmdList("where %s" % depotPath)
 250     output = None
 251     for entry in outputList:
 252         if entry["depotFile"] == depotPath:
 253             output = entry
 254             break
 255     if output == None:
 256         return ""
 257     if output["code"] == "error":
 258         return ""
 259     clientPath = ""
 260     if "path" in output:
 261         clientPath = output.get("path")
 262     elif "data" in output:
 263         data = output.get("data")
 264         lastSpace = data.rfind(" ")
 265         clientPath = data[lastSpace + 1:]
 266
 267     if clientPath.endswith("..."):
 268         clientPath = clientPath[:-3]
 269     return clientPath
 270
 271 def currentGitBranch():
 272     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 273
 274 def isValidGitDir(path):
 275     if (os.path.exists(path + "/HEAD")
 276         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 277         return True;
 278     return False
 279
 280 def parseRevision(ref):
 281     return read_pipe("git rev-parse %s" % ref).strip()
 282
 283 def extractLogMessageFromGitCommit(commit):
 284     logMessage = ""
 285
 286     ## fixme: title is first line of commit, not 1st paragraph.
 287     foundTitle = False
 288     for log in read_pipe_lines("git cat-file commit %s" % commit):
 289        if not foundTitle:
 290            if len(log) == 1:
 291                foundTitle = True
 292            continue
 293
 294        logMessage += log
 295     return logMessage
 296
 297 def extractSettingsGitLog(log):
 298     values = {}
 299     for line in log.split("\n"):
 300         line = line.strip()
 301         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 302         if not m:
 303             continue
 304
 305         assignments = m.group(1).split (':')
 306         for a in assignments:
 307             vals = a.split ('=')
 308             key = vals[0].strip()
 309             val = ('='.join (vals[1:])).strip()
 310             if val.endswith ('\"') and val.startswith('"'):
 311                 val = val[1:-1]
 312
 313             values[key] = val
 314
 315     paths = values.get("depot-paths")
 316     if not paths:
 317         paths = values.get("depot-path")
 318     if paths:
 319         values['depot-paths'] = paths.split(',')
 320     return values
 321
 322 def gitBranchExists(branch):
 323     proc = subprocess.Popen(["git", "rev-parse", branch],
 324                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 325     return proc.wait() == 0;
 326
 327 _gitConfig = {}
 328 def gitConfig(key):
 329     if not _gitConfig.has_key(key):
 330         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 331     return _gitConfig[key]
 332
 333 def p4BranchesInGit(branchesAreInRemotes = True):
 334     branches = {}
 335
 336     cmdline = "git rev-parse --symbolic "
 337     if branchesAreInRemotes:
 338         cmdline += " --remotes"
 339     else:
 340         cmdline += " --branches"
 341
 342     for line in read_pipe_lines(cmdline):
 343         line = line.strip()
 344
 345         ## only import to p4/
 346         if not line.startswith('p4/') or line == "p4/HEAD":
 347             continue
 348         branch = line
 349
 350         # strip off p4
 351         branch = re.sub ("^p4/", "", line)
 352
 353         branches[branch] = parseRevision(line)
 354     return branches
 355
 356 def findUpstreamBranchPoint(head = "HEAD"):
 357     branches = p4BranchesInGit()
 358     # map from depot-path to branch name
 359     branchByDepotPath = {}
 360     for branch in branches.keys():
 361         tip = branches[branch]
 362         log = extractLogMessageFromGitCommit(tip)
 363         settings = extractSettingsGitLog(log)
 364         if settings.has_key("depot-paths"):
 365             paths = ",".join(settings["depot-paths"])
 366             branchByDepotPath[paths] = "remotes/p4/" + branch
 367
 368     settings = None
 369     parent = 0
 370     while parent < 65535:
 371         commit = head + "~%s" % parent
 372         log = extractLogMessageFromGitCommit(commit)
 373         settings = extractSettingsGitLog(log)
 374         if settings.has_key("depot-paths"):
 375             paths = ",".join(settings["depot-paths"])
 376             if branchByDepotPath.has_key(paths):
 377                 return [branchByDepotPath[paths], settings]
 378
 379         parent = parent + 1
 380
 381     return ["", settings]
 382
 383 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 384     if not silent:
 385         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 386                % localRefPrefix)
 387
 388     originPrefix = "origin/p4/"
 389
 390     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 391         line = line.strip()
 392         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 393             continue
 394
 395         headName = line[len(originPrefix):]
 396         remoteHead = localRefPrefix + headName
 397         originHead = line
 398
 399         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 400         if (not original.has_key('depot-paths')
 401             or not original.has_key('change')):
 402             continue
 403
 404         update = False
 405         if not gitBranchExists(remoteHead):
 406             if verbose:
 407                 print "creating %s" % remoteHead
 408             update = True
 409         else:
 410             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 411             if settings.has_key('change') > 0:
 412                 if settings['depot-paths'] == original['depot-paths']:
 413                     originP4Change = int(original['change'])
 414                     p4Change = int(settings['change'])
 415                     if originP4Change > p4Change:
 416                         print ("%s (%s) is newer than %s (%s). "
 417                                "Updating p4 branch from origin."
 418                                % (originHead, originP4Change,
 419                                   remoteHead, p4Change))
 420                         update = True
 421                 else:
 422                     print ("Ignoring: %s was imported from %s while "
 423                            "%s was imported from %s"
 424                            % (originHead, ','.join(original['depot-paths']),
 425                               remoteHead, ','.join(settings['depot-paths'])))
 426
 427         if update:
 428             system("git update-ref %s %s" % (remoteHead, originHead))
 429
 430 def originP4BranchesExist():
 431         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 432
 433 def p4ChangesForPaths(depotPaths, changeRange):
 434     assert depotPaths
 435     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 436                                                         for p in depotPaths]))
 437
 438     changes = []
 439     for line in output:
 440         changeNum = line.split(" ")[1]
 441         changes.append(int(changeNum))
 442
 443     changes.sort()
 444     return changes
 445
 446 class Command:
 447     def __init__(self):
 448         self.usage = "usage: %prog [options]"
 449         self.needsGit = True
 450
 451 class P4Debug(Command):
 452     def __init__(self):
 453         Command.__init__(self)
 454         self.options = [
 455             optparse.make_option("--verbose", dest="verbose", action="store_true",
 456                                  default=False),
 457             ]
 458         self.description = "A tool to debug the output of p4 -G."
 459         self.needsGit = False
 460         self.verbose = False
 461
 462     def run(self, args):
 463         j = 0
 464         for output in p4CmdList(" ".join(args)):
 465             print 'Element: %d' % j
 466             j += 1
 467             print output
 468         return True
 469
 470 class P4RollBack(Command):
 471     def __init__(self):
 472         Command.__init__(self)
 473         self.options = [
 474             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 475             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 476         ]
 477         self.description = "A tool to debug the multi-branch import. Don't use :)"
 478         self.verbose = False
 479         self.rollbackLocalBranches = False
 480
 481     def run(self, args):
 482         if len(args) != 1:
 483             return False
 484         maxChange = int(args[0])
 485
 486         if "p4ExitCode" in p4Cmd("changes -m 1"):
 487             die("Problems executing p4");
 488
 489         if self.rollbackLocalBranches:
 490             refPrefix = "refs/heads/"
 491             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 492         else:
 493             refPrefix = "refs/remotes/"
 494             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 495
 496         for line in lines:
 497             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 498                 line = line.strip()
 499                 ref = refPrefix + line
 500                 log = extractLogMessageFromGitCommit(ref)
 501                 settings = extractSettingsGitLog(log)
 502
 503                 depotPaths = settings['depot-paths']
 504                 change = settings['change']
 505
 506                 changed = False
 507
 508                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 509                                                            for p in depotPaths]))) == 0:
 510                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 511                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 512                     continue
 513
 514                 while change and int(change) > maxChange:
 515                     changed = True
 516                     if self.verbose:
 517                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 518                     system("git update-ref %s \"%s^\"" % (ref, ref))
 519                     log = extractLogMessageFromGitCommit(ref)
 520                     settings =  extractSettingsGitLog(log)
 521
 522
 523                     depotPaths = settings['depot-paths']
 524                     change = settings['change']
 525
 526                 if changed:
 527                     print "%s rewound to %s" % (ref, change)
 528
 529         return True
 530
 531 class P4Submit(Command):
 532     def __init__(self):
 533         Command.__init__(self)
 534         self.options = [
 535                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 536                 optparse.make_option("--origin", dest="origin"),
 537                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 538         ]
 539         self.description = "Submit changes from git to the perforce depot."
 540         self.usage += " [name of git branch to submit into perforce depot]"
 541         self.interactive = True
 542         self.origin = ""
 543         self.detectRename = False
 544         self.verbose = False
 545         self.isWindows = (platform.system() == "Windows")
 546
 547     def check(self):
 548         if len(p4CmdList("opened ...")) > 0:
 549             die("You have files opened with perforce! Close them before starting the sync.")
 550
 551     # replaces everything between 'Description:' and the next P4 submit template field with the
 552     # commit message
 553     def prepareLogMessage(self, template, message):
 554         result = ""
 555
 556         inDescriptionSection = False
 557
 558         for line in template.split("\n"):
 559             if line.startswith("#"):
 560                 result += line + "\n"
 561                 continue
 562
 563             if inDescriptionSection:
 564                 if line.startswith("Files:"):
 565                     inDescriptionSection = False
 566                 else:
 567                     continue
 568             else:
 569                 if line.startswith("Description:"):
 570                     inDescriptionSection = True
 571                     line += "\n"
 572                     for messageLine in message.split("\n"):
 573                         line += "\t" + messageLine + "\n"
 574
 575             result += line + "\n"
 576
 577         return result
 578
 579     def prepareSubmitTemplate(self):
 580         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 581         template = ""
 582         inFilesSection = False
 583         for line in p4_read_pipe_lines("change -o"):
 584             if line.endswith("\r\n"):
 585                 line = line[:-2] + "\n"
 586             if inFilesSection:
 587                 if line.startswith("\t"):
 588                     # path starts and ends with a tab
 589                     path = line[1:]
 590                     lastTab = path.rfind("\t")
 591                     if lastTab != -1:
 592                         path = path[:lastTab]
 593                         if not path.startswith(self.depotPath):
 594                             continue
 595                 else:
 596                     inFilesSection = False
 597             else:
 598                 if line.startswith("Files:"):
 599                     inFilesSection = True
 600
 601             template += line
 602
 603         return template
 604
 605     def applyCommit(self, id):
 606         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 607         diffOpts = ("", "-M")[self.detectRename]
 608         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 609         filesToAdd = set()
 610         filesToDelete = set()
 611         editedFiles = set()
 612         filesToChangeExecBit = {}
 613         for line in diff:
 614             diff = parseDiffTreeEntry(line)
 615             modifier = diff['status']
 616             path = diff['src']
 617             if modifier == "M":
 618                 p4_system("edit \"%s\"" % path)
 619                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 620                     filesToChangeExecBit[path] = diff['dst_mode']
 621                 editedFiles.add(path)
 622             elif modifier == "A":
 623                 filesToAdd.add(path)
 624                 filesToChangeExecBit[path] = diff['dst_mode']
 625                 if path in filesToDelete:
 626                     filesToDelete.remove(path)
 627             elif modifier == "D":
 628                 filesToDelete.add(path)
 629                 if path in filesToAdd:
 630                     filesToAdd.remove(path)
 631             elif modifier == "R":
 632                 src, dest = diff['src'], diff['dst']
 633                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 634                 p4_system("edit \"%s\"" % (dest))
 635                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 636                     filesToChangeExecBit[dest] = diff['dst_mode']
 637                 os.unlink(dest)
 638                 editedFiles.add(dest)
 639                 filesToDelete.add(src)
 640             else:
 641                 die("unknown modifier %s for %s" % (modifier, path))
 642
 643         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 644         patchcmd = diffcmd + " | git apply "
 645         tryPatchCmd = patchcmd + "--check -"
 646         applyPatchCmd = patchcmd + "--check --apply -"
 647
 648         if os.system(tryPatchCmd) != 0:
 649             print "Unfortunately applying the change failed!"
 650             print "What do you want to do?"
 651             response = "x"
 652             while response != "s" and response != "a" and response != "w":
 653                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 654                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 655             if response == "s":
 656                 print "Skipping! Good luck with the next patches..."
 657                 for f in editedFiles:
 658                     p4_system("revert \"%s\"" % f);
 659                 for f in filesToAdd:
 660                     system("rm %s" %f)
 661                 return
 662             elif response == "a":
 663                 os.system(applyPatchCmd)
 664                 if len(filesToAdd) > 0:
 665                     print "You may also want to call p4 add on the following files:"
 666                     print " ".join(filesToAdd)
 667                 if len(filesToDelete):
 668                     print "The following files should be scheduled for deletion with p4 delete:"
 669                     print " ".join(filesToDelete)
 670                 die("Please resolve and submit the conflict manually and "
 671                     + "continue afterwards with git-p4 submit --continue")
 672             elif response == "w":
 673                 system(diffcmd + " > patch.txt")
 674                 print "Patch saved to patch.txt in %s !" % self.clientPath
 675                 die("Please resolve and submit the conflict manually and "
 676                     "continue afterwards with git-p4 submit --continue")
 677
 678         system(applyPatchCmd)
 679
 680         for f in filesToAdd:
 681             p4_system("add \"%s\"" % f)
 682         for f in filesToDelete:
 683             p4_system("revert \"%s\"" % f)
 684             p4_system("delete \"%s\"" % f)
 685
 686         # Set/clear executable bits
 687         for f in filesToChangeExecBit.keys():
 688             mode = filesToChangeExecBit[f]
 689             setP4ExecBit(f, mode)
 690
 691         logMessage = extractLogMessageFromGitCommit(id)
 692         logMessage = logMessage.strip()
 693
 694         template = self.prepareSubmitTemplate()
 695
 696         if self.interactive:
 697             submitTemplate = self.prepareLogMessage(template, logMessage)
 698             if os.environ.has_key("P4DIFF"):
 699                 del(os.environ["P4DIFF"])
 700             diff = p4_read_pipe("diff -du ...")
 701
 702             newdiff = ""
 703             for newFile in filesToAdd:
 704                 newdiff += "==== new file ====\n"
 705                 newdiff += "--- /dev/null\n"
 706                 newdiff += "+++ %s\n" % newFile
 707                 f = open(newFile, "r")
 708                 for line in f.readlines():
 709                     newdiff += "+" + line
 710                 f.close()
 711
 712             separatorLine = "######## everything below this line is just the diff #######\n"
 713
 714             [handle, fileName] = tempfile.mkstemp()
 715             tmpFile = os.fdopen(handle, "w+")
 716             if self.isWindows:
 717                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 718                 separatorLine = separatorLine.replace("\n", "\r\n")
 719                 newdiff = newdiff.replace("\n", "\r\n")
 720             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 721             tmpFile.close()
 722             mtime = os.stat(fileName).st_mtime
 723             defaultEditor = "vi"
 724             if platform.system() == "Windows":
 725                 defaultEditor = "notepad"
 726             if os.environ.has_key("P4EDITOR"):
 727                 editor = os.environ.get("P4EDITOR")
 728             else:
 729                 editor = os.environ.get("EDITOR", defaultEditor);
 730             system(editor + " " + fileName)
 731
 732             response = "y"
 733             if os.stat(fileName).st_mtime <= mtime:
 734                 response = "x"
 735                 while response != "y" and response != "n":
 736                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 737
 738             if response == "y":
 739                 tmpFile = open(fileName, "rb")
 740                 message = tmpFile.read()
 741                 tmpFile.close()
 742                 submitTemplate = message[:message.index(separatorLine)]
 743                 if self.isWindows:
 744                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 745                 p4_write_pipe("submit -i", submitTemplate)
 746             else:
 747                 for f in editedFiles:
 748                     p4_system("revert \"%s\"" % f);
 749                 for f in filesToAdd:
 750                     p4_system("revert \"%s\"" % f);
 751                     system("rm %s" %f)
 752
 753             os.remove(fileName)
 754         else:
 755             fileName = "submit.txt"
 756             file = open(fileName, "w+")
 757             file.write(self.prepareLogMessage(template, logMessage))
 758             file.close()
 759             print ("Perforce submit template written as %s. "
 760                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 761                    % (fileName, fileName))
 762
 763     def run(self, args):
 764         if len(args) == 0:
 765             self.master = currentGitBranch()
 766             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 767                 die("Detecting current git branch failed!")
 768         elif len(args) == 1:
 769             self.master = args[0]
 770         else:
 771             return False
 772
 773         allowSubmit = gitConfig("git-p4.allowSubmit")
 774         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 775             die("%s is not in git-p4.allowSubmit" % self.master)
 776
 777         [upstream, settings] = findUpstreamBranchPoint()
 778         self.depotPath = settings['depot-paths'][0]
 779         if len(self.origin) == 0:
 780             self.origin = upstream
 781
 782         if self.verbose:
 783             print "Origin branch is " + self.origin
 784
 785         if len(self.depotPath) == 0:
 786             print "Internal error: cannot locate perforce depot path from existing branches"
 787             sys.exit(128)
 788
 789         self.clientPath = p4Where(self.depotPath)
 790
 791         if len(self.clientPath) == 0:
 792             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 793             sys.exit(128)
 794
 795         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 796         self.oldWorkingDirectory = os.getcwd()
 797
 798         chdir(self.clientPath)
 799         print "Syncronizing p4 checkout..."
 800         p4_system("sync ...")
 801
 802         self.check()
 803
 804         commits = []
 805         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 806             commits.append(line.strip())
 807         commits.reverse()
 808
 809         while len(commits) > 0:
 810             commit = commits[0]
 811             commits = commits[1:]
 812             self.applyCommit(commit)
 813             if not self.interactive:
 814                 break
 815
 816         if len(commits) == 0:
 817             print "All changes applied!"
 818             chdir(self.oldWorkingDirectory)
 819
 820             sync = P4Sync()
 821             sync.run([])
 822
 823             rebase = P4Rebase()
 824             rebase.rebase()
 825
 826         return True
 827
 828 class P4Sync(Command):
 829     def __init__(self):
 830         Command.__init__(self)
 831         self.options = [
 832                 optparse.make_option("--branch", dest="branch"),
 833                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 834                 optparse.make_option("--changesfile", dest="changesFile"),
 835                 optparse.make_option("--silent", dest="silent", action="store_true"),
 836                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 837                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 838                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 839                                      help="Import into refs/heads/ , not refs/remotes"),
 840                 optparse.make_option("--max-changes", dest="maxChanges"),
 841                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 842                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 843                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 844                                      help="Only sync files that are included in the Perforce Client Spec")
 845         ]
 846         self.description = """Imports from Perforce into a git repository.\n
 847     example:
 848     //depot/my/project/ -- to import the current head
 849     //depot/my/project/@all -- to import everything
 850     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 851
 852     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 853
 854         self.usage += " //depot/path[@revRange]"
 855         self.silent = False
 856         self.createdBranches = Set()
 857         self.committedChanges = Set()
 858         self.branch = ""
 859         self.detectBranches = False
 860         self.detectLabels = False
 861         self.changesFile = ""
 862         self.syncWithOrigin = True
 863         self.verbose = False
 864         self.importIntoRemotes = True
 865         self.maxChanges = ""
 866         self.isWindows = (platform.system() == "Windows")
 867         self.keepRepoPath = False
 868         self.depotPaths = None
 869         self.p4BranchesInGit = []
 870         self.cloneExclude = []
 871         self.useClientSpec = False
 872         self.clientSpecDirs = []
 873
 874         if gitConfig("git-p4.syncFromOrigin") == "false":
 875             self.syncWithOrigin = False
 876
 877     def extractFilesFromCommit(self, commit):
 878         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 879                              for path in self.cloneExclude]
 880         files = []
 881         fnum = 0
 882         while commit.has_key("depotFile%s" % fnum):
 883             path =  commit["depotFile%s" % fnum]
 884
 885             if [p for p in self.cloneExclude
 886                 if path.startswith (p)]:
 887                 found = False
 888             else:
 889                 found = [p for p in self.depotPaths
 890                          if path.startswith (p)]
 891             if not found:
 892                 fnum = fnum + 1
 893                 continue
 894
 895             file = {}
 896             file["path"] = path
 897             file["rev"] = commit["rev%s" % fnum]
 898             file["action"] = commit["action%s" % fnum]
 899             file["type"] = commit["type%s" % fnum]
 900             files.append(file)
 901             fnum = fnum + 1
 902         return files
 903
 904     def stripRepoPath(self, path, prefixes):
 905         if self.keepRepoPath:
 906             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 907
 908         for p in prefixes:
 909             if path.startswith(p):
 910                 path = path[len(p):]
 911
 912         return path
 913
 914     def splitFilesIntoBranches(self, commit):
 915         branches = {}
 916         fnum = 0
 917         while commit.has_key("depotFile%s" % fnum):
 918             path =  commit["depotFile%s" % fnum]
 919             found = [p for p in self.depotPaths
 920                      if path.startswith (p)]
 921             if not found:
 922                 fnum = fnum + 1
 923                 continue
 924
 925             file = {}
 926             file["path"] = path
 927             file["rev"] = commit["rev%s" % fnum]
 928             file["action"] = commit["action%s" % fnum]
 929             file["type"] = commit["type%s" % fnum]
 930             fnum = fnum + 1
 931
 932             relPath = self.stripRepoPath(path, self.depotPaths)
 933
 934             for branch in self.knownBranches.keys():
 935
 936                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 937                 if relPath.startswith(branch + "/"):
 938                     if branch not in branches:
 939                         branches[branch] = []
 940                     branches[branch].append(file)
 941                     break
 942
 943         return branches
 944
 945     ## Should move this out, doesn't use SELF.
 946     def readP4Files(self, files):
 947         filesForCommit = []
 948         filesToRead = []
 949
 950         for f in files:
 951             includeFile = True
 952             for val in self.clientSpecDirs:
 953                 if f['path'].startswith(val[0]):
 954                     if val[1] <= 0:
 955                         includeFile = False
 956                     break
 957
 958             if includeFile:
 959                 filesForCommit.append(f)
 960                 if f['action'] not in ('delete', 'purge'):
 961                     filesToRead.append(f)
 962
 963         filedata = []
 964         if len(filesToRead) > 0:
 965             filedata = p4CmdList('-x - print',
 966                                  stdin='\n'.join(['%s#%s' % (f['path'], f['rev'])
 967                                                   for f in filesToRead]),
 968                                  stdin_mode='w+')
 969
 970             if "p4ExitCode" in filedata[0]:
 971                 die("Problems executing p4. Error: [%d]."
 972                     % (filedata[0]['p4ExitCode']));
 973
 974         j = 0;
 975         contents = {}
 976         while j < len(filedata):
 977             stat = filedata[j]
 978             j += 1
 979             text = ''
 980             while j < len(filedata) and filedata[j]['code'] in ('text', 'unicode', 'binary'):
 981                 text += filedata[j]['data']
 982                 del filedata[j]['data']
 983                 j += 1
 984
 985             if not stat.has_key('depotFile'):
 986                 sys.stderr.write("p4 print fails with: %s\n" % repr(stat))
 987                 continue
 988
 989             if stat['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
 990                 text = re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text)
 991             elif stat['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
 992                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text)
 993
 994             contents[stat['depotFile']] = text
 995
 996         for f in filesForCommit:
 997             path = f['path']
 998             if contents.has_key(path):
 999                 f['data'] = contents[path]
1000
1001         return filesForCommit
1002
1003     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1004         epoch = details["time"]
1005         author = details["user"]
1006
1007         if self.verbose:
1008             print "commit into %s" % branch
1009
1010         # start with reading files; if that fails, we should not
1011         # create a commit.
1012         new_files = []
1013         for f in files:
1014             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1015                 new_files.append (f)
1016             else:
1017                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1018         files = self.readP4Files(new_files)
1019
1020         self.gitStream.write("commit %s\n" % branch)
1021 #        gitStream.write("mark :%s\n" % details["change"])
1022         self.committedChanges.add(int(details["change"]))
1023         committer = ""
1024         if author not in self.users:
1025             self.getUserMapFromPerforceServer()
1026         if author in self.users:
1027             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1028         else:
1029             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1030
1031         self.gitStream.write("committer %s\n" % committer)
1032
1033         self.gitStream.write("data <<EOT\n")
1034         self.gitStream.write(details["desc"])
1035         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1036                              % (','.join (branchPrefixes), details["change"]))
1037         if len(details['options']) > 0:
1038             self.gitStream.write(": options = %s" % details['options'])
1039         self.gitStream.write("]\nEOT\n\n")
1040
1041         if len(parent) > 0:
1042             if self.verbose:
1043                 print "parent %s" % parent
1044             self.gitStream.write("from %s\n" % parent)
1045
1046         for file in files:
1047             if file["type"] == "apple":
1048                 print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path']
1049                 continue
1050
1051             relPath = self.stripRepoPath(file['path'], branchPrefixes)
1052             if file["action"] in ("delete", "purge"):
1053                 self.gitStream.write("D %s\n" % relPath)
1054             else:
1055                 data = file['data']
1056
1057                 mode = "644"
1058                 if isP4Exec(file["type"]):
1059                     mode = "755"
1060                 elif file["type"] == "symlink":
1061                     mode = "120000"
1062                     # p4 print on a symlink contains "target\n", so strip it off
1063                     data = data[:-1]
1064
1065                 if self.isWindows and file["type"].endswith("text"):
1066                     data = data.replace("\r\n", "\n")
1067
1068                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1069                 self.gitStream.write("data %s\n" % len(data))
1070                 self.gitStream.write(data)
1071                 self.gitStream.write("\n")
1072
1073         self.gitStream.write("\n")
1074
1075         change = int(details["change"])
1076
1077         if self.labels.has_key(change):
1078             label = self.labels[change]
1079             labelDetails = label[0]
1080             labelRevisions = label[1]
1081             if self.verbose:
1082                 print "Change %s is labelled %s" % (change, labelDetails)
1083
1084             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1085                                                     for p in branchPrefixes]))
1086
1087             if len(files) == len(labelRevisions):
1088
1089                 cleanedFiles = {}
1090                 for info in files:
1091                     if info["action"] in ("delete", "purge"):
1092                         continue
1093                     cleanedFiles[info["depotFile"]] = info["rev"]
1094
1095                 if cleanedFiles == labelRevisions:
1096                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1097                     self.gitStream.write("from %s\n" % branch)
1098
1099                     owner = labelDetails["Owner"]
1100                     tagger = ""
1101                     if author in self.users:
1102                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1103                     else:
1104                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1105                     self.gitStream.write("tagger %s\n" % tagger)
1106                     self.gitStream.write("data <<EOT\n")
1107                     self.gitStream.write(labelDetails["Description"])
1108                     self.gitStream.write("EOT\n\n")
1109
1110                 else:
1111                     if not self.silent:
1112                         print ("Tag %s does not match with change %s: files do not match."
1113                                % (labelDetails["label"], change))
1114
1115             else:
1116                 if not self.silent:
1117                     print ("Tag %s does not match with change %s: file count is different."
1118                            % (labelDetails["label"], change))
1119
1120     def getUserCacheFilename(self):
1121         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1122         return home + "/.gitp4-usercache.txt"
1123
1124     def getUserMapFromPerforceServer(self):
1125         if self.userMapFromPerforceServer:
1126             return
1127         self.users = {}
1128
1129         for output in p4CmdList("users"):
1130             if not output.has_key("User"):
1131                 continue
1132             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1133
1134
1135         s = ''
1136         for (key, val) in self.users.items():
1137             s += "%s\t%s\n" % (key, val)
1138
1139         open(self.getUserCacheFilename(), "wb").write(s)
1140         self.userMapFromPerforceServer = True
1141
1142     def loadUserMapFromCache(self):
1143         self.users = {}
1144         self.userMapFromPerforceServer = False
1145         try:
1146             cache = open(self.getUserCacheFilename(), "rb")
1147             lines = cache.readlines()
1148             cache.close()
1149             for line in lines:
1150                 entry = line.strip().split("\t")
1151                 self.users[entry[0]] = entry[1]
1152         except IOError:
1153             self.getUserMapFromPerforceServer()
1154
1155     def getLabels(self):
1156         self.labels = {}
1157
1158         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1159         if len(l) > 0 and not self.silent:
1160             print "Finding files belonging to labels in %s" % `self.depotPaths`
1161
1162         for output in l:
1163             label = output["label"]
1164             revisions = {}
1165             newestChange = 0
1166             if self.verbose:
1167                 print "Querying files for label %s" % label
1168             for file in p4CmdList("files "
1169                                   +  ' '.join (["%s...@%s" % (p, label)
1170                                                 for p in self.depotPaths])):
1171                 revisions[file["depotFile"]] = file["rev"]
1172                 change = int(file["change"])
1173                 if change > newestChange:
1174                     newestChange = change
1175
1176             self.labels[newestChange] = [output, revisions]
1177
1178         if self.verbose:
1179             print "Label changes: %s" % self.labels.keys()
1180
1181     def guessProjectName(self):
1182         for p in self.depotPaths:
1183             if p.endswith("/"):
1184                 p = p[:-1]
1185             p = p[p.strip().rfind("/") + 1:]
1186             if not p.endswith("/"):
1187                p += "/"
1188             return p
1189
1190     def getBranchMapping(self):
1191         lostAndFoundBranches = set()
1192
1193         for info in p4CmdList("branches"):
1194             details = p4Cmd("branch -o %s" % info["branch"])
1195             viewIdx = 0
1196             while details.has_key("View%s" % viewIdx):
1197                 paths = details["View%s" % viewIdx].split(" ")
1198                 viewIdx = viewIdx + 1
1199                 # require standard //depot/foo/... //depot/bar/... mapping
1200                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1201                     continue
1202                 source = paths[0]
1203                 destination = paths[1]
1204                 ## HACK
1205                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1206                     source = source[len(self.depotPaths[0]):-4]
1207                     destination = destination[len(self.depotPaths[0]):-4]
1208
1209                     if destination in self.knownBranches:
1210                         if not self.silent:
1211                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1212                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1213                         continue
1214
1215                     self.knownBranches[destination] = source
1216
1217                     lostAndFoundBranches.discard(destination)
1218
1219                     if source not in self.knownBranches:
1220                         lostAndFoundBranches.add(source)
1221
1222
1223         for branch in lostAndFoundBranches:
1224             self.knownBranches[branch] = branch
1225
1226     def getBranchMappingFromGitBranches(self):
1227         branches = p4BranchesInGit(self.importIntoRemotes)
1228         for branch in branches.keys():
1229             if branch == "master":
1230                 branch = "main"
1231             else:
1232                 branch = branch[len(self.projectName):]
1233             self.knownBranches[branch] = branch
1234
1235     def listExistingP4GitBranches(self):
1236         # branches holds mapping from name to commit
1237         branches = p4BranchesInGit(self.importIntoRemotes)
1238         self.p4BranchesInGit = branches.keys()
1239         for branch in branches.keys():
1240             self.initialParents[self.refPrefix + branch] = branches[branch]
1241
1242     def updateOptionDict(self, d):
1243         option_keys = {}
1244         if self.keepRepoPath:
1245             option_keys['keepRepoPath'] = 1
1246
1247         d["options"] = ' '.join(sorted(option_keys.keys()))
1248
1249     def readOptions(self, d):
1250         self.keepRepoPath = (d.has_key('options')
1251                              and ('keepRepoPath' in d['options']))
1252
1253     def gitRefForBranch(self, branch):
1254         if branch == "main":
1255             return self.refPrefix + "master"
1256
1257         if len(branch) <= 0:
1258             return branch
1259
1260         return self.refPrefix + self.projectName + branch
1261
1262     def gitCommitByP4Change(self, ref, change):
1263         if self.verbose:
1264             print "looking in ref " + ref + " for change %s using bisect..." % change
1265
1266         earliestCommit = ""
1267         latestCommit = parseRevision(ref)
1268
1269         while True:
1270             if self.verbose:
1271                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1272             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1273             if len(next) == 0:
1274                 if self.verbose:
1275                     print "argh"
1276                 return ""
1277             log = extractLogMessageFromGitCommit(next)
1278             settings = extractSettingsGitLog(log)
1279             currentChange = int(settings['change'])
1280             if self.verbose:
1281                 print "current change %s" % currentChange
1282
1283             if currentChange == change:
1284                 if self.verbose:
1285                     print "found %s" % next
1286                 return next
1287
1288             if currentChange < change:
1289                 earliestCommit = "^%s" % next
1290             else:
1291                 latestCommit = "%s" % next
1292
1293         return ""
1294
1295     def importNewBranch(self, branch, maxChange):
1296         # make fast-import flush all changes to disk and update the refs using the checkpoint
1297         # command so that we can try to find the branch parent in the git history
1298         self.gitStream.write("checkpoint\n\n");
1299         self.gitStream.flush();
1300         branchPrefix = self.depotPaths[0] + branch + "/"
1301         range = "@1,%s" % maxChange
1302         #print "prefix" + branchPrefix
1303         changes = p4ChangesForPaths([branchPrefix], range)
1304         if len(changes) <= 0:
1305             return False
1306         firstChange = changes[0]
1307         #print "first change in branch: %s" % firstChange
1308         sourceBranch = self.knownBranches[branch]
1309         sourceDepotPath = self.depotPaths[0] + sourceBranch
1310         sourceRef = self.gitRefForBranch(sourceBranch)
1311         #print "source " + sourceBranch
1312
1313         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1314         #print "branch parent: %s" % branchParentChange
1315         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1316         if len(gitParent) > 0:
1317             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1318             #print "parent git commit: %s" % gitParent
1319
1320         self.importChanges(changes)
1321         return True
1322
1323     def importChanges(self, changes):
1324         cnt = 1
1325         for change in changes:
1326             description = p4Cmd("describe %s" % change)
1327             self.updateOptionDict(description)
1328
1329             if not self.silent:
1330                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1331                 sys.stdout.flush()
1332             cnt = cnt + 1
1333
1334             try:
1335                 if self.detectBranches:
1336                     branches = self.splitFilesIntoBranches(description)
1337                     for branch in branches.keys():
1338                         ## HACK  --hwn
1339                         branchPrefix = self.depotPaths[0] + branch + "/"
1340
1341                         parent = ""
1342
1343                         filesForCommit = branches[branch]
1344
1345                         if self.verbose:
1346                             print "branch is %s" % branch
1347
1348                         self.updatedBranches.add(branch)
1349
1350                         if branch not in self.createdBranches:
1351                             self.createdBranches.add(branch)
1352                             parent = self.knownBranches[branch]
1353                             if parent == branch:
1354                                 parent = ""
1355                             else:
1356                                 fullBranch = self.projectName + branch
1357                                 if fullBranch not in self.p4BranchesInGit:
1358                                     if not self.silent:
1359                                         print("\n    Importing new branch %s" % fullBranch);
1360                                     if self.importNewBranch(branch, change - 1):
1361                                         parent = ""
1362                                         self.p4BranchesInGit.append(fullBranch)
1363                                     if not self.silent:
1364                                         print("\n    Resuming with change %s" % change);
1365
1366                                 if self.verbose:
1367                                     print "parent determined through known branches: %s" % parent
1368
1369                         branch = self.gitRefForBranch(branch)
1370                         parent = self.gitRefForBranch(parent)
1371
1372                         if self.verbose:
1373                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1374
1375                         if len(parent) == 0 and branch in self.initialParents:
1376                             parent = self.initialParents[branch]
1377                             del self.initialParents[branch]
1378
1379                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1380                 else:
1381                     files = self.extractFilesFromCommit(description)
1382                     self.commit(description, files, self.branch, self.depotPaths,
1383                                 self.initialParent)
1384                     self.initialParent = ""
1385             except IOError:
1386                 print self.gitError.read()
1387                 sys.exit(1)
1388
1389     def importHeadRevision(self, revision):
1390         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1391
1392         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1393         details["desc"] = ("Initial import of %s from the state at revision %s"
1394                            % (' '.join(self.depotPaths), revision))
1395         details["change"] = revision
1396         newestRevision = 0
1397
1398         fileCnt = 0
1399         for info in p4CmdList("files "
1400                               +  ' '.join(["%s...%s"
1401                                            % (p, revision)
1402                                            for p in self.depotPaths])):
1403
1404             if info['code'] == 'error':
1405                 sys.stderr.write("p4 returned an error: %s\n"
1406                                  % info['data'])
1407                 sys.exit(1)
1408
1409
1410             change = int(info["change"])
1411             if change > newestRevision:
1412                 newestRevision = change
1413
1414             if info["action"] in ("delete", "purge"):
1415                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1416                 #fileCnt = fileCnt + 1
1417                 continue
1418
1419             for prop in ["depotFile", "rev", "action", "type" ]:
1420                 details["%s%s" % (prop, fileCnt)] = info[prop]
1421
1422             fileCnt = fileCnt + 1
1423
1424         details["change"] = newestRevision
1425         self.updateOptionDict(details)
1426         try:
1427             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1428         except IOError:
1429             print "IO error with git fast-import. Is your git version recent enough?"
1430             print self.gitError.read()
1431
1432
1433     def getClientSpec(self):
1434         specList = p4CmdList( "client -o" )
1435         temp = {}
1436         for entry in specList:
1437             for k,v in entry.iteritems():
1438                 if k.startswith("View"):
1439                     if v.startswith('"'):
1440                         start = 1
1441                     else:
1442                         start = 0
1443                     index = v.find("...")
1444                     v = v[start:index]
1445                     if v.startswith("-"):
1446                         v = v[1:]
1447                         temp[v] = -len(v)
1448                     else:
1449                         temp[v] = len(v)
1450         self.clientSpecDirs = temp.items()
1451         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1452
1453     def run(self, args):
1454         self.depotPaths = []
1455         self.changeRange = ""
1456         self.initialParent = ""
1457         self.previousDepotPaths = []
1458
1459         # map from branch depot path to parent branch
1460         self.knownBranches = {}
1461         self.initialParents = {}
1462         self.hasOrigin = originP4BranchesExist()
1463         if not self.syncWithOrigin:
1464             self.hasOrigin = False
1465
1466         if self.importIntoRemotes:
1467             self.refPrefix = "refs/remotes/p4/"
1468         else:
1469             self.refPrefix = "refs/heads/p4/"
1470
1471         if self.syncWithOrigin and self.hasOrigin:
1472             if not self.silent:
1473                 print "Syncing with origin first by calling git fetch origin"
1474             system("git fetch origin")
1475
1476         if len(self.branch) == 0:
1477             self.branch = self.refPrefix + "master"
1478             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1479                 system("git update-ref %s refs/heads/p4" % self.branch)
1480                 system("git branch -D p4");
1481             # create it /after/ importing, when master exists
1482             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1483                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1484
1485         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1486             self.getClientSpec()
1487
1488         # TODO: should always look at previous commits,
1489         # merge with previous imports, if possible.
1490         if args == []:
1491             if self.hasOrigin:
1492                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1493             self.listExistingP4GitBranches()
1494
1495             if len(self.p4BranchesInGit) > 1:
1496                 if not self.silent:
1497                     print "Importing from/into multiple branches"
1498                 self.detectBranches = True
1499
1500             if self.verbose:
1501                 print "branches: %s" % self.p4BranchesInGit
1502
1503             p4Change = 0
1504             for branch in self.p4BranchesInGit:
1505                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1506
1507                 settings = extractSettingsGitLog(logMsg)
1508
1509                 self.readOptions(settings)
1510                 if (settings.has_key('depot-paths')
1511                     and settings.has_key ('change')):
1512                     change = int(settings['change']) + 1
1513                     p4Change = max(p4Change, change)
1514
1515                     depotPaths = sorted(settings['depot-paths'])
1516                     if self.previousDepotPaths == []:
1517                         self.previousDepotPaths = depotPaths
1518                     else:
1519                         paths = []
1520                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1521                             for i in range(0, min(len(cur), len(prev))):
1522                                 if cur[i] <> prev[i]:
1523                                     i = i - 1
1524                                     break
1525
1526                             paths.append (cur[:i + 1])
1527
1528                         self.previousDepotPaths = paths
1529
1530             if p4Change > 0:
1531                 self.depotPaths = sorted(self.previousDepotPaths)
1532                 self.changeRange = "@%s,#head" % p4Change
1533                 if not self.detectBranches:
1534                     self.initialParent = parseRevision(self.branch)
1535                 if not self.silent and not self.detectBranches:
1536                     print "Performing incremental import into %s git branch" % self.branch
1537
1538         if not self.branch.startswith("refs/"):
1539             self.branch = "refs/heads/" + self.branch
1540
1541         if len(args) == 0 and self.depotPaths:
1542             if not self.silent:
1543                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1544         else:
1545             if self.depotPaths and self.depotPaths != args:
1546                 print ("previous import used depot path %s and now %s was specified. "
1547                        "This doesn't work!" % (' '.join (self.depotPaths),
1548                                                ' '.join (args)))
1549                 sys.exit(1)
1550
1551             self.depotPaths = sorted(args)
1552
1553         revision = ""
1554         self.users = {}
1555
1556         newPaths = []
1557         for p in self.depotPaths:
1558             if p.find("@") != -1:
1559                 atIdx = p.index("@")
1560                 self.changeRange = p[atIdx:]
1561                 if self.changeRange == "@all":
1562                     self.changeRange = ""
1563                 elif ',' not in self.changeRange:
1564                     revision = self.changeRange
1565                     self.changeRange = ""
1566                 p = p[:atIdx]
1567             elif p.find("#") != -1:
1568                 hashIdx = p.index("#")
1569                 revision = p[hashIdx:]
1570                 p = p[:hashIdx]
1571             elif self.previousDepotPaths == []:
1572                 revision = "#head"
1573
1574             p = re.sub ("\.\.\.$", "", p)
1575             if not p.endswith("/"):
1576                 p += "/"
1577
1578             newPaths.append(p)
1579
1580         self.depotPaths = newPaths
1581
1582
1583         self.loadUserMapFromCache()
1584         self.labels = {}
1585         if self.detectLabels:
1586             self.getLabels();
1587
1588         if self.detectBranches:
1589             ## FIXME - what's a P4 projectName ?
1590             self.projectName = self.guessProjectName()
1591
1592             if self.hasOrigin:
1593                 self.getBranchMappingFromGitBranches()
1594             else:
1595                 self.getBranchMapping()
1596             if self.verbose:
1597                 print "p4-git branches: %s" % self.p4BranchesInGit
1598                 print "initial parents: %s" % self.initialParents
1599             for b in self.p4BranchesInGit:
1600                 if b != "master":
1601
1602                     ## FIXME
1603                     b = b[len(self.projectName):]
1604                 self.createdBranches.add(b)
1605
1606         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1607
1608         importProcess = subprocess.Popen(["git", "fast-import"],
1609                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1610                                          stderr=subprocess.PIPE);
1611         self.gitOutput = importProcess.stdout
1612         self.gitStream = importProcess.stdin
1613         self.gitError = importProcess.stderr
1614
1615         if revision:
1616             self.importHeadRevision(revision)
1617         else:
1618             changes = []
1619
1620             if len(self.changesFile) > 0:
1621                 output = open(self.changesFile).readlines()
1622                 changeSet = Set()
1623                 for line in output:
1624                     changeSet.add(int(line))
1625
1626                 for change in changeSet:
1627                     changes.append(change)
1628
1629                 changes.sort()
1630             else:
1631                 if self.verbose:
1632                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1633                                                               self.changeRange)
1634                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1635
1636                 if len(self.maxChanges) > 0:
1637                     changes = changes[:min(int(self.maxChanges), len(changes))]
1638
1639             if len(changes) == 0:
1640                 if not self.silent:
1641                     print "No changes to import!"
1642                 return True
1643
1644             if not self.silent and not self.detectBranches:
1645                 print "Import destination: %s" % self.branch
1646
1647             self.updatedBranches = set()
1648
1649             self.importChanges(changes)
1650
1651             if not self.silent:
1652                 print ""
1653                 if len(self.updatedBranches) > 0:
1654                     sys.stdout.write("Updated branches: ")
1655                     for b in self.updatedBranches:
1656                         sys.stdout.write("%s " % b)
1657                     sys.stdout.write("\n")
1658
1659         self.gitStream.close()
1660         if importProcess.wait() != 0:
1661             die("fast-import failed: %s" % self.gitError.read())
1662         self.gitOutput.close()
1663         self.gitError.close()
1664
1665         return True
1666
1667 class P4Rebase(Command):
1668     def __init__(self):
1669         Command.__init__(self)
1670         self.options = [ ]
1671         self.description = ("Fetches the latest revision from perforce and "
1672                             + "rebases the current work (branch) against it")
1673         self.verbose = False
1674
1675     def run(self, args):
1676         sync = P4Sync()
1677         sync.run([])
1678
1679         return self.rebase()
1680
1681     def rebase(self):
1682         if os.system("git update-index --refresh") != 0:
1683             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1684         if len(read_pipe("git diff-index HEAD --")) > 0:
1685             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1686
1687         [upstream, settings] = findUpstreamBranchPoint()
1688         if len(upstream) == 0:
1689             die("Cannot find upstream branchpoint for rebase")
1690
1691         # the branchpoint may be p4/foo~3, so strip off the parent
1692         upstream = re.sub("~[0-9]+$", "", upstream)
1693
1694         print "Rebasing the current branch onto %s" % upstream
1695         oldHead = read_pipe("git rev-parse HEAD").strip()
1696         system("git rebase %s" % upstream)
1697         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1698         return True
1699
1700 class P4Clone(P4Sync):
1701     def __init__(self):
1702         P4Sync.__init__(self)
1703         self.description = "Creates a new git repository and imports from Perforce into it"
1704         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1705         self.options += [
1706             optparse.make_option("--destination", dest="cloneDestination",
1707                                  action='store', default=None,
1708                                  help="where to leave result of the clone"),
1709             optparse.make_option("-/", dest="cloneExclude",
1710                                  action="append", type="string",
1711                                  help="exclude depot path")
1712         ]
1713         self.cloneDestination = None
1714         self.needsGit = False
1715
1716     # This is required for the "append" cloneExclude action
1717     def ensure_value(self, attr, value):
1718         if not hasattr(self, attr) or getattr(self, attr) is None:
1719             setattr(self, attr, value)
1720         return getattr(self, attr)
1721
1722     def defaultDestination(self, args):
1723         ## TODO: use common prefix of args?
1724         depotPath = args[0]
1725         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1726         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1727         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1728         depotDir = re.sub(r"/$", "", depotDir)
1729         return os.path.split(depotDir)[1]
1730
1731     def run(self, args):
1732         if len(args) < 1:
1733             return False
1734
1735         if self.keepRepoPath and not self.cloneDestination:
1736             sys.stderr.write("Must specify destination for --keep-path\n")
1737             sys.exit(1)
1738
1739         depotPaths = args
1740
1741         if not self.cloneDestination and len(depotPaths) > 1:
1742             self.cloneDestination = depotPaths[-1]
1743             depotPaths = depotPaths[:-1]
1744
1745         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1746         for p in depotPaths:
1747             if not p.startswith("//"):
1748                 return False
1749
1750         if not self.cloneDestination:
1751             self.cloneDestination = self.defaultDestination(args)
1752
1753         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1754         if not os.path.exists(self.cloneDestination):
1755             os.makedirs(self.cloneDestination)
1756         chdir(self.cloneDestination)
1757         system("git init")
1758         self.gitdir = os.getcwd() + "/.git"
1759         if not P4Sync.run(self, depotPaths):
1760             return False
1761         if self.branch != "master":
1762             if self.importIntoRemotes:
1763                 masterbranch = "refs/remotes/p4/master"
1764             else:
1765                 masterbranch = "refs/heads/p4/master"
1766             if gitBranchExists(masterbranch):
1767                 system("git branch master %s" % masterbranch)
1768                 system("git checkout -f")
1769             else:
1770                 print "Could not detect main branch. No checkout/master branch created."
1771
1772         return True
1773
1774 class P4Branches(Command):
1775     def __init__(self):
1776         Command.__init__(self)
1777         self.options = [ ]
1778         self.description = ("Shows the git branches that hold imports and their "
1779                             + "corresponding perforce depot paths")
1780         self.verbose = False
1781
1782     def run(self, args):
1783         if originP4BranchesExist():
1784             createOrUpdateBranchesFromOrigin()
1785
1786         cmdline = "git rev-parse --symbolic "
1787         cmdline += " --remotes"
1788
1789         for line in read_pipe_lines(cmdline):
1790             line = line.strip()
1791
1792             if not line.startswith('p4/') or line == "p4/HEAD":
1793                 continue
1794             branch = line
1795
1796             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1797             settings = extractSettingsGitLog(log)
1798
1799             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1800         return True
1801
1802 class HelpFormatter(optparse.IndentedHelpFormatter):
1803     def __init__(self):
1804         optparse.IndentedHelpFormatter.__init__(self)
1805
1806     def format_description(self, description):
1807         if description:
1808             return description + "\n"
1809         else:
1810             return ""
1811
1812 def printUsage(commands):
1813     print "usage: %s <command> [options]" % sys.argv[0]
1814     print ""
1815     print "valid commands: %s" % ", ".join(commands)
1816     print ""
1817     print "Try %s <command> --help for command specific help." % sys.argv[0]
1818     print ""
1819
1820 commands = {
1821     "debug" : P4Debug,
1822     "submit" : P4Submit,
1823     "commit" : P4Submit,
1824     "sync" : P4Sync,
1825     "rebase" : P4Rebase,
1826     "clone" : P4Clone,
1827     "rollback" : P4RollBack,
1828     "branches" : P4Branches
1829 }
1830
1831
1832 def main():
1833     if len(sys.argv[1:]) == 0:
1834         printUsage(commands.keys())
1835         sys.exit(2)
1836
1837     cmd = ""
1838     cmdName = sys.argv[1]
1839     try:
1840         klass = commands[cmdName]
1841         cmd = klass()
1842     except KeyError:
1843         print "unknown command %s" % cmdName
1844         print ""
1845         printUsage(commands.keys())
1846         sys.exit(2)
1847
1848     options = cmd.options
1849     cmd.gitdir = os.environ.get("GIT_DIR", None)
1850
1851     args = sys.argv[2:]
1852
1853     if len(options) > 0:
1854         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1855
1856         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1857                                        options,
1858                                        description = cmd.description,
1859                                        formatter = HelpFormatter())
1860
1861         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1862     global verbose
1863     verbose = cmd.verbose
1864     if cmd.needsGit:
1865         if cmd.gitdir == None:
1866             cmd.gitdir = os.path.abspath(".git")
1867             if not isValidGitDir(cmd.gitdir):
1868                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1869                 if os.path.exists(cmd.gitdir):
1870                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1871                     if len(cdup) > 0:
1872                         chdir(cdup);
1873
1874         if not isValidGitDir(cmd.gitdir):
1875             if isValidGitDir(cmd.gitdir + "/.git"):
1876                 cmd.gitdir += "/.git"
1877             else:
1878                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1879
1880         os.environ["GIT_DIR"] = cmd.gitdir
1881
1882     if not cmd.run(args):
1883         parser.print_help()
1884
1885
1886 if __name__ == '__main__':
1887     main()