contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key):
 337     if not _gitConfig.has_key(key):
 338         _gitConfig[key] = read_pipe("git config %s" % key, ignore_error=True).strip()
 339     return _gitConfig[key]
 340
 341 def p4BranchesInGit(branchesAreInRemotes = True):
 342     branches = {}
 343
 344     cmdline = "git rev-parse --symbolic "
 345     if branchesAreInRemotes:
 346         cmdline += " --remotes"
 347     else:
 348         cmdline += " --branches"
 349
 350     for line in read_pipe_lines(cmdline):
 351         line = line.strip()
 352
 353         ## only import to p4/
 354         if not line.startswith('p4/') or line == "p4/HEAD":
 355             continue
 356         branch = line
 357
 358         # strip off p4
 359         branch = re.sub ("^p4/", "", line)
 360
 361         branches[branch] = parseRevision(line)
 362     return branches
 363
 364 def findUpstreamBranchPoint(head = "HEAD"):
 365     branches = p4BranchesInGit()
 366     # map from depot-path to branch name
 367     branchByDepotPath = {}
 368     for branch in branches.keys():
 369         tip = branches[branch]
 370         log = extractLogMessageFromGitCommit(tip)
 371         settings = extractSettingsGitLog(log)
 372         if settings.has_key("depot-paths"):
 373             paths = ",".join(settings["depot-paths"])
 374             branchByDepotPath[paths] = "remotes/p4/" + branch
 375
 376     settings = None
 377     parent = 0
 378     while parent < 65535:
 379         commit = head + "~%s" % parent
 380         log = extractLogMessageFromGitCommit(commit)
 381         settings = extractSettingsGitLog(log)
 382         if settings.has_key("depot-paths"):
 383             paths = ",".join(settings["depot-paths"])
 384             if branchByDepotPath.has_key(paths):
 385                 return [branchByDepotPath[paths], settings]
 386
 387         parent = parent + 1
 388
 389     return ["", settings]
 390
 391 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 392     if not silent:
 393         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 394                % localRefPrefix)
 395
 396     originPrefix = "origin/p4/"
 397
 398     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 399         line = line.strip()
 400         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 401             continue
 402
 403         headName = line[len(originPrefix):]
 404         remoteHead = localRefPrefix + headName
 405         originHead = line
 406
 407         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 408         if (not original.has_key('depot-paths')
 409             or not original.has_key('change')):
 410             continue
 411
 412         update = False
 413         if not gitBranchExists(remoteHead):
 414             if verbose:
 415                 print "creating %s" % remoteHead
 416             update = True
 417         else:
 418             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 419             if settings.has_key('change') > 0:
 420                 if settings['depot-paths'] == original['depot-paths']:
 421                     originP4Change = int(original['change'])
 422                     p4Change = int(settings['change'])
 423                     if originP4Change > p4Change:
 424                         print ("%s (%s) is newer than %s (%s). "
 425                                "Updating p4 branch from origin."
 426                                % (originHead, originP4Change,
 427                                   remoteHead, p4Change))
 428                         update = True
 429                 else:
 430                     print ("Ignoring: %s was imported from %s while "
 431                            "%s was imported from %s"
 432                            % (originHead, ','.join(original['depot-paths']),
 433                               remoteHead, ','.join(settings['depot-paths'])))
 434
 435         if update:
 436             system("git update-ref %s %s" % (remoteHead, originHead))
 437
 438 def originP4BranchesExist():
 439         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 440
 441 def p4ChangesForPaths(depotPaths, changeRange):
 442     assert depotPaths
 443     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 444                                                         for p in depotPaths]))
 445
 446     changes = {}
 447     for line in output:
 448         changeNum = int(line.split(" ")[1])
 449         changes[changeNum] = True
 450
 451     changelist = changes.keys()
 452     changelist.sort()
 453     return changelist
 454
 455 class Command:
 456     def __init__(self):
 457         self.usage = "usage: %prog [options]"
 458         self.needsGit = True
 459
 460 class P4Debug(Command):
 461     def __init__(self):
 462         Command.__init__(self)
 463         self.options = [
 464             optparse.make_option("--verbose", dest="verbose", action="store_true",
 465                                  default=False),
 466             ]
 467         self.description = "A tool to debug the output of p4 -G."
 468         self.needsGit = False
 469         self.verbose = False
 470
 471     def run(self, args):
 472         j = 0
 473         for output in p4CmdList(" ".join(args)):
 474             print 'Element: %d' % j
 475             j += 1
 476             print output
 477         return True
 478
 479 class P4RollBack(Command):
 480     def __init__(self):
 481         Command.__init__(self)
 482         self.options = [
 483             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 484             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 485         ]
 486         self.description = "A tool to debug the multi-branch import. Don't use :)"
 487         self.verbose = False
 488         self.rollbackLocalBranches = False
 489
 490     def run(self, args):
 491         if len(args) != 1:
 492             return False
 493         maxChange = int(args[0])
 494
 495         if "p4ExitCode" in p4Cmd("changes -m 1"):
 496             die("Problems executing p4");
 497
 498         if self.rollbackLocalBranches:
 499             refPrefix = "refs/heads/"
 500             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 501         else:
 502             refPrefix = "refs/remotes/"
 503             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 504
 505         for line in lines:
 506             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 507                 line = line.strip()
 508                 ref = refPrefix + line
 509                 log = extractLogMessageFromGitCommit(ref)
 510                 settings = extractSettingsGitLog(log)
 511
 512                 depotPaths = settings['depot-paths']
 513                 change = settings['change']
 514
 515                 changed = False
 516
 517                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 518                                                            for p in depotPaths]))) == 0:
 519                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 520                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 521                     continue
 522
 523                 while change and int(change) > maxChange:
 524                     changed = True
 525                     if self.verbose:
 526                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 527                     system("git update-ref %s \"%s^\"" % (ref, ref))
 528                     log = extractLogMessageFromGitCommit(ref)
 529                     settings =  extractSettingsGitLog(log)
 530
 531
 532                     depotPaths = settings['depot-paths']
 533                     change = settings['change']
 534
 535                 if changed:
 536                     print "%s rewound to %s" % (ref, change)
 537
 538         return True
 539
 540 class P4Submit(Command):
 541     def __init__(self):
 542         Command.__init__(self)
 543         self.options = [
 544                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 545                 optparse.make_option("--origin", dest="origin"),
 546                 optparse.make_option("-M", dest="detectRename", action="store_true"),
 547         ]
 548         self.description = "Submit changes from git to the perforce depot."
 549         self.usage += " [name of git branch to submit into perforce depot]"
 550         self.interactive = True
 551         self.origin = ""
 552         self.detectRename = False
 553         self.verbose = False
 554         self.isWindows = (platform.system() == "Windows")
 555
 556     def check(self):
 557         if len(p4CmdList("opened ...")) > 0:
 558             die("You have files opened with perforce! Close them before starting the sync.")
 559
 560     # replaces everything between 'Description:' and the next P4 submit template field with the
 561     # commit message
 562     def prepareLogMessage(self, template, message):
 563         result = ""
 564
 565         inDescriptionSection = False
 566
 567         for line in template.split("\n"):
 568             if line.startswith("#"):
 569                 result += line + "\n"
 570                 continue
 571
 572             if inDescriptionSection:
 573                 if line.startswith("Files:"):
 574                     inDescriptionSection = False
 575                 else:
 576                     continue
 577             else:
 578                 if line.startswith("Description:"):
 579                     inDescriptionSection = True
 580                     line += "\n"
 581                     for messageLine in message.split("\n"):
 582                         line += "\t" + messageLine + "\n"
 583
 584             result += line + "\n"
 585
 586         return result
 587
 588     def prepareSubmitTemplate(self):
 589         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 590         template = ""
 591         inFilesSection = False
 592         for line in p4_read_pipe_lines("change -o"):
 593             if line.endswith("\r\n"):
 594                 line = line[:-2] + "\n"
 595             if inFilesSection:
 596                 if line.startswith("\t"):
 597                     # path starts and ends with a tab
 598                     path = line[1:]
 599                     lastTab = path.rfind("\t")
 600                     if lastTab != -1:
 601                         path = path[:lastTab]
 602                         if not path.startswith(self.depotPath):
 603                             continue
 604                 else:
 605                     inFilesSection = False
 606             else:
 607                 if line.startswith("Files:"):
 608                     inFilesSection = True
 609
 610             template += line
 611
 612         return template
 613
 614     def applyCommit(self, id):
 615         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 616         diffOpts = ("", "-M")[self.detectRename]
 617         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 618         filesToAdd = set()
 619         filesToDelete = set()
 620         editedFiles = set()
 621         filesToChangeExecBit = {}
 622         for line in diff:
 623             diff = parseDiffTreeEntry(line)
 624             modifier = diff['status']
 625             path = diff['src']
 626             if modifier == "M":
 627                 p4_system("edit \"%s\"" % path)
 628                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 629                     filesToChangeExecBit[path] = diff['dst_mode']
 630                 editedFiles.add(path)
 631             elif modifier == "A":
 632                 filesToAdd.add(path)
 633                 filesToChangeExecBit[path] = diff['dst_mode']
 634                 if path in filesToDelete:
 635                     filesToDelete.remove(path)
 636             elif modifier == "D":
 637                 filesToDelete.add(path)
 638                 if path in filesToAdd:
 639                     filesToAdd.remove(path)
 640             elif modifier == "R":
 641                 src, dest = diff['src'], diff['dst']
 642                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 643                 p4_system("edit \"%s\"" % (dest))
 644                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 645                     filesToChangeExecBit[dest] = diff['dst_mode']
 646                 os.unlink(dest)
 647                 editedFiles.add(dest)
 648                 filesToDelete.add(src)
 649             else:
 650                 die("unknown modifier %s for %s" % (modifier, path))
 651
 652         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 653         patchcmd = diffcmd + " | git apply "
 654         tryPatchCmd = patchcmd + "--check -"
 655         applyPatchCmd = patchcmd + "--check --apply -"
 656
 657         if os.system(tryPatchCmd) != 0:
 658             print "Unfortunately applying the change failed!"
 659             print "What do you want to do?"
 660             response = "x"
 661             while response != "s" and response != "a" and response != "w":
 662                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 663                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 664             if response == "s":
 665                 print "Skipping! Good luck with the next patches..."
 666                 for f in editedFiles:
 667                     p4_system("revert \"%s\"" % f);
 668                 for f in filesToAdd:
 669                     system("rm %s" %f)
 670                 return
 671             elif response == "a":
 672                 os.system(applyPatchCmd)
 673                 if len(filesToAdd) > 0:
 674                     print "You may also want to call p4 add on the following files:"
 675                     print " ".join(filesToAdd)
 676                 if len(filesToDelete):
 677                     print "The following files should be scheduled for deletion with p4 delete:"
 678                     print " ".join(filesToDelete)
 679                 die("Please resolve and submit the conflict manually and "
 680                     + "continue afterwards with git-p4 submit --continue")
 681             elif response == "w":
 682                 system(diffcmd + " > patch.txt")
 683                 print "Patch saved to patch.txt in %s !" % self.clientPath
 684                 die("Please resolve and submit the conflict manually and "
 685                     "continue afterwards with git-p4 submit --continue")
 686
 687         system(applyPatchCmd)
 688
 689         for f in filesToAdd:
 690             p4_system("add \"%s\"" % f)
 691         for f in filesToDelete:
 692             p4_system("revert \"%s\"" % f)
 693             p4_system("delete \"%s\"" % f)
 694
 695         # Set/clear executable bits
 696         for f in filesToChangeExecBit.keys():
 697             mode = filesToChangeExecBit[f]
 698             setP4ExecBit(f, mode)
 699
 700         logMessage = extractLogMessageFromGitCommit(id)
 701         logMessage = logMessage.strip()
 702
 703         template = self.prepareSubmitTemplate()
 704
 705         if self.interactive:
 706             submitTemplate = self.prepareLogMessage(template, logMessage)
 707             if os.environ.has_key("P4DIFF"):
 708                 del(os.environ["P4DIFF"])
 709             diff = ""
 710             for editedFile in editedFiles:
 711                 diff += p4_read_pipe("diff -du %r" % editedFile)
 712
 713             newdiff = ""
 714             for newFile in filesToAdd:
 715                 newdiff += "==== new file ====\n"
 716                 newdiff += "--- /dev/null\n"
 717                 newdiff += "+++ %s\n" % newFile
 718                 f = open(newFile, "r")
 719                 for line in f.readlines():
 720                     newdiff += "+" + line
 721                 f.close()
 722
 723             separatorLine = "######## everything below this line is just the diff #######\n"
 724
 725             [handle, fileName] = tempfile.mkstemp()
 726             tmpFile = os.fdopen(handle, "w+")
 727             if self.isWindows:
 728                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 729                 separatorLine = separatorLine.replace("\n", "\r\n")
 730                 newdiff = newdiff.replace("\n", "\r\n")
 731             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 732             tmpFile.close()
 733             mtime = os.stat(fileName).st_mtime
 734             if os.environ.has_key("P4EDITOR"):
 735                 editor = os.environ.get("P4EDITOR")
 736             else:
 737                 editor = read_pipe("git var GIT_EDITOR").strip()
 738             system(editor + " " + fileName)
 739
 740             response = "y"
 741             if os.stat(fileName).st_mtime <= mtime:
 742                 response = "x"
 743                 while response != "y" and response != "n":
 744                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 745
 746             if response == "y":
 747                 tmpFile = open(fileName, "rb")
 748                 message = tmpFile.read()
 749                 tmpFile.close()
 750                 submitTemplate = message[:message.index(separatorLine)]
 751                 if self.isWindows:
 752                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 753                 p4_write_pipe("submit -i", submitTemplate)
 754             else:
 755                 for f in editedFiles:
 756                     p4_system("revert \"%s\"" % f);
 757                 for f in filesToAdd:
 758                     p4_system("revert \"%s\"" % f);
 759                     system("rm %s" %f)
 760
 761             os.remove(fileName)
 762         else:
 763             fileName = "submit.txt"
 764             file = open(fileName, "w+")
 765             file.write(self.prepareLogMessage(template, logMessage))
 766             file.close()
 767             print ("Perforce submit template written as %s. "
 768                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 769                    % (fileName, fileName))
 770
 771     def run(self, args):
 772         if len(args) == 0:
 773             self.master = currentGitBranch()
 774             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 775                 die("Detecting current git branch failed!")
 776         elif len(args) == 1:
 777             self.master = args[0]
 778         else:
 779             return False
 780
 781         allowSubmit = gitConfig("git-p4.allowSubmit")
 782         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 783             die("%s is not in git-p4.allowSubmit" % self.master)
 784
 785         [upstream, settings] = findUpstreamBranchPoint()
 786         self.depotPath = settings['depot-paths'][0]
 787         if len(self.origin) == 0:
 788             self.origin = upstream
 789
 790         if self.verbose:
 791             print "Origin branch is " + self.origin
 792
 793         if len(self.depotPath) == 0:
 794             print "Internal error: cannot locate perforce depot path from existing branches"
 795             sys.exit(128)
 796
 797         self.clientPath = p4Where(self.depotPath)
 798
 799         if len(self.clientPath) == 0:
 800             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
 801             sys.exit(128)
 802
 803         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
 804         self.oldWorkingDirectory = os.getcwd()
 805
 806         chdir(self.clientPath)
 807         print "Synchronizing p4 checkout..."
 808         p4_system("sync ...")
 809
 810         self.check()
 811
 812         commits = []
 813         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
 814             commits.append(line.strip())
 815         commits.reverse()
 816
 817         while len(commits) > 0:
 818             commit = commits[0]
 819             commits = commits[1:]
 820             self.applyCommit(commit)
 821             if not self.interactive:
 822                 break
 823
 824         if len(commits) == 0:
 825             print "All changes applied!"
 826             chdir(self.oldWorkingDirectory)
 827
 828             sync = P4Sync()
 829             sync.run([])
 830
 831             rebase = P4Rebase()
 832             rebase.rebase()
 833
 834         return True
 835
 836 class P4Sync(Command):
 837     delete_actions = ( "delete", "move/delete", "purge" )
 838
 839     def __init__(self):
 840         Command.__init__(self)
 841         self.options = [
 842                 optparse.make_option("--branch", dest="branch"),
 843                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 844                 optparse.make_option("--changesfile", dest="changesFile"),
 845                 optparse.make_option("--silent", dest="silent", action="store_true"),
 846                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
 847                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 848                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
 849                                      help="Import into refs/heads/ , not refs/remotes"),
 850                 optparse.make_option("--max-changes", dest="maxChanges"),
 851                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
 852                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
 853                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
 854                                      help="Only sync files that are included in the Perforce Client Spec")
 855         ]
 856         self.description = """Imports from Perforce into a git repository.\n
 857     example:
 858     //depot/my/project/ -- to import the current head
 859     //depot/my/project/@all -- to import everything
 860     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 861
 862     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 863
 864         self.usage += " //depot/path[@revRange]"
 865         self.silent = False
 866         self.createdBranches = set()
 867         self.committedChanges = set()
 868         self.branch = ""
 869         self.detectBranches = False
 870         self.detectLabels = False
 871         self.changesFile = ""
 872         self.syncWithOrigin = True
 873         self.verbose = False
 874         self.importIntoRemotes = True
 875         self.maxChanges = ""
 876         self.isWindows = (platform.system() == "Windows")
 877         self.keepRepoPath = False
 878         self.depotPaths = None
 879         self.p4BranchesInGit = []
 880         self.cloneExclude = []
 881         self.useClientSpec = False
 882         self.clientSpecDirs = []
 883
 884         if gitConfig("git-p4.syncFromOrigin") == "false":
 885             self.syncWithOrigin = False
 886
 887     #
 888     # P4 wildcards are not allowed in filenames.  P4 complains
 889     # if you simply add them, but you can force it with "-f", in
 890     # which case it translates them into %xx encoding internally.
 891     # Search for and fix just these four characters.  Do % last so
 892     # that fixing it does not inadvertently create new %-escapes.
 893     #
 894     def wildcard_decode(self, path):
 895         # Cannot have * in a filename in windows; untested as to
 896         # what p4 would do in such a case.
 897         if not self.isWindows:
 898             path = path.replace("%2A", "*")
 899         path = path.replace("%23", "#") \
 900                    .replace("%40", "@") \
 901                    .replace("%25", "%")
 902         return path
 903
 904     def extractFilesFromCommit(self, commit):
 905         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
 906                              for path in self.cloneExclude]
 907         files = []
 908         fnum = 0
 909         while commit.has_key("depotFile%s" % fnum):
 910             path =  commit["depotFile%s" % fnum]
 911
 912             if [p for p in self.cloneExclude
 913                 if path.startswith (p)]:
 914                 found = False
 915             else:
 916                 found = [p for p in self.depotPaths
 917                          if path.startswith (p)]
 918             if not found:
 919                 fnum = fnum + 1
 920                 continue
 921
 922             file = {}
 923             file["path"] = path
 924             file["rev"] = commit["rev%s" % fnum]
 925             file["action"] = commit["action%s" % fnum]
 926             file["type"] = commit["type%s" % fnum]
 927             files.append(file)
 928             fnum = fnum + 1
 929         return files
 930
 931     def stripRepoPath(self, path, prefixes):
 932         if self.keepRepoPath:
 933             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
 934
 935         for p in prefixes:
 936             if path.startswith(p):
 937                 path = path[len(p):]
 938
 939         return path
 940
 941     def splitFilesIntoBranches(self, commit):
 942         branches = {}
 943         fnum = 0
 944         while commit.has_key("depotFile%s" % fnum):
 945             path =  commit["depotFile%s" % fnum]
 946             found = [p for p in self.depotPaths
 947                      if path.startswith (p)]
 948             if not found:
 949                 fnum = fnum + 1
 950                 continue
 951
 952             file = {}
 953             file["path"] = path
 954             file["rev"] = commit["rev%s" % fnum]
 955             file["action"] = commit["action%s" % fnum]
 956             file["type"] = commit["type%s" % fnum]
 957             fnum = fnum + 1
 958
 959             relPath = self.stripRepoPath(path, self.depotPaths)
 960
 961             for branch in self.knownBranches.keys():
 962
 963                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
 964                 if relPath.startswith(branch + "/"):
 965                     if branch not in branches:
 966                         branches[branch] = []
 967                     branches[branch].append(file)
 968                     break
 969
 970         return branches
 971
 972     # output one file from the P4 stream
 973     # - helper for streamP4Files
 974
 975     def streamOneP4File(self, file, contents):
 976         if file["type"] == "apple":
 977             print "\nfile %s is a strange apple file that forks. Ignoring" % \
 978                 file['depotFile']
 979             return
 980
 981         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
 982         relPath = self.wildcard_decode(relPath)
 983         if verbose:
 984             sys.stderr.write("%s\n" % relPath)
 985
 986         mode = "644"
 987         if isP4Exec(file["type"]):
 988             mode = "755"
 989         elif file["type"] == "symlink":
 990             mode = "120000"
 991             # p4 print on a symlink contains "target\n", so strip it off
 992             data = ''.join(contents)
 993             contents = [data[:-1]]
 994
 995         if self.isWindows and file["type"].endswith("text"):
 996             mangled = []
 997             for data in contents:
 998                 data = data.replace("\r\n", "\n")
 999                 mangled.append(data)
1000             contents = mangled
1001
1002         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1003             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1004         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1005             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1006
1007         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1008
1009         # total length...
1010         length = 0
1011         for d in contents:
1012             length = length + len(d)
1013
1014         self.gitStream.write("data %d\n" % length)
1015         for d in contents:
1016             self.gitStream.write(d)
1017         self.gitStream.write("\n")
1018
1019     def streamOneP4Deletion(self, file):
1020         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1021         if verbose:
1022             sys.stderr.write("delete %s\n" % relPath)
1023         self.gitStream.write("D %s\n" % relPath)
1024
1025     # handle another chunk of streaming data
1026     def streamP4FilesCb(self, marshalled):
1027
1028         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1029             # start of a new file - output the old one first
1030             self.streamOneP4File(self.stream_file, self.stream_contents)
1031             self.stream_file = {}
1032             self.stream_contents = []
1033             self.stream_have_file_info = False
1034
1035         # pick up the new file information... for the
1036         # 'data' field we need to append to our array
1037         for k in marshalled.keys():
1038             if k == 'data':
1039                 self.stream_contents.append(marshalled['data'])
1040             else:
1041                 self.stream_file[k] = marshalled[k]
1042
1043         self.stream_have_file_info = True
1044
1045     # Stream directly from "p4 files" into "git fast-import"
1046     def streamP4Files(self, files):
1047         filesForCommit = []
1048         filesToRead = []
1049         filesToDelete = []
1050
1051         for f in files:
1052             includeFile = True
1053             for val in self.clientSpecDirs:
1054                 if f['path'].startswith(val[0]):
1055                     if val[1] <= 0:
1056                         includeFile = False
1057                     break
1058
1059             if includeFile:
1060                 filesForCommit.append(f)
1061                 if f['action'] in self.delete_actions:
1062                     filesToDelete.append(f)
1063                 else:
1064                     filesToRead.append(f)
1065
1066         # deleted files...
1067         for f in filesToDelete:
1068             self.streamOneP4Deletion(f)
1069
1070         if len(filesToRead) > 0:
1071             self.stream_file = {}
1072             self.stream_contents = []
1073             self.stream_have_file_info = False
1074
1075             # curry self argument
1076             def streamP4FilesCbSelf(entry):
1077                 self.streamP4FilesCb(entry)
1078
1079             p4CmdList("-x - print",
1080                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1081                                                   for f in filesToRead]),
1082                 cb=streamP4FilesCbSelf)
1083
1084             # do the last chunk
1085             if self.stream_file.has_key('depotFile'):
1086                 self.streamOneP4File(self.stream_file, self.stream_contents)
1087
1088     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1089         epoch = details["time"]
1090         author = details["user"]
1091         self.branchPrefixes = branchPrefixes
1092
1093         if self.verbose:
1094             print "commit into %s" % branch
1095
1096         # start with reading files; if that fails, we should not
1097         # create a commit.
1098         new_files = []
1099         for f in files:
1100             if [p for p in branchPrefixes if f['path'].startswith(p)]:
1101                 new_files.append (f)
1102             else:
1103                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % path)
1104
1105         self.gitStream.write("commit %s\n" % branch)
1106 #        gitStream.write("mark :%s\n" % details["change"])
1107         self.committedChanges.add(int(details["change"]))
1108         committer = ""
1109         if author not in self.users:
1110             self.getUserMapFromPerforceServer()
1111         if author in self.users:
1112             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1113         else:
1114             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1115
1116         self.gitStream.write("committer %s\n" % committer)
1117
1118         self.gitStream.write("data <<EOT\n")
1119         self.gitStream.write(details["desc"])
1120         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1121                              % (','.join (branchPrefixes), details["change"]))
1122         if len(details['options']) > 0:
1123             self.gitStream.write(": options = %s" % details['options'])
1124         self.gitStream.write("]\nEOT\n\n")
1125
1126         if len(parent) > 0:
1127             if self.verbose:
1128                 print "parent %s" % parent
1129             self.gitStream.write("from %s\n" % parent)
1130
1131         self.streamP4Files(new_files)
1132         self.gitStream.write("\n")
1133
1134         change = int(details["change"])
1135
1136         if self.labels.has_key(change):
1137             label = self.labels[change]
1138             labelDetails = label[0]
1139             labelRevisions = label[1]
1140             if self.verbose:
1141                 print "Change %s is labelled %s" % (change, labelDetails)
1142
1143             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1144                                                     for p in branchPrefixes]))
1145
1146             if len(files) == len(labelRevisions):
1147
1148                 cleanedFiles = {}
1149                 for info in files:
1150                     if info["action"] in self.delete_actions:
1151                         continue
1152                     cleanedFiles[info["depotFile"]] = info["rev"]
1153
1154                 if cleanedFiles == labelRevisions:
1155                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1156                     self.gitStream.write("from %s\n" % branch)
1157
1158                     owner = labelDetails["Owner"]
1159                     tagger = ""
1160                     if author in self.users:
1161                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1162                     else:
1163                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1164                     self.gitStream.write("tagger %s\n" % tagger)
1165                     self.gitStream.write("data <<EOT\n")
1166                     self.gitStream.write(labelDetails["Description"])
1167                     self.gitStream.write("EOT\n\n")
1168
1169                 else:
1170                     if not self.silent:
1171                         print ("Tag %s does not match with change %s: files do not match."
1172                                % (labelDetails["label"], change))
1173
1174             else:
1175                 if not self.silent:
1176                     print ("Tag %s does not match with change %s: file count is different."
1177                            % (labelDetails["label"], change))
1178
1179     def getUserCacheFilename(self):
1180         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
1181         return home + "/.gitp4-usercache.txt"
1182
1183     def getUserMapFromPerforceServer(self):
1184         if self.userMapFromPerforceServer:
1185             return
1186         self.users = {}
1187
1188         for output in p4CmdList("users"):
1189             if not output.has_key("User"):
1190                 continue
1191             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
1192
1193
1194         s = ''
1195         for (key, val) in self.users.items():
1196             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
1197
1198         open(self.getUserCacheFilename(), "wb").write(s)
1199         self.userMapFromPerforceServer = True
1200
1201     def loadUserMapFromCache(self):
1202         self.users = {}
1203         self.userMapFromPerforceServer = False
1204         try:
1205             cache = open(self.getUserCacheFilename(), "rb")
1206             lines = cache.readlines()
1207             cache.close()
1208             for line in lines:
1209                 entry = line.strip().split("\t")
1210                 self.users[entry[0]] = entry[1]
1211         except IOError:
1212             self.getUserMapFromPerforceServer()
1213
1214     def getLabels(self):
1215         self.labels = {}
1216
1217         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1218         if len(l) > 0 and not self.silent:
1219             print "Finding files belonging to labels in %s" % `self.depotPaths`
1220
1221         for output in l:
1222             label = output["label"]
1223             revisions = {}
1224             newestChange = 0
1225             if self.verbose:
1226                 print "Querying files for label %s" % label
1227             for file in p4CmdList("files "
1228                                   +  ' '.join (["%s...@%s" % (p, label)
1229                                                 for p in self.depotPaths])):
1230                 revisions[file["depotFile"]] = file["rev"]
1231                 change = int(file["change"])
1232                 if change > newestChange:
1233                     newestChange = change
1234
1235             self.labels[newestChange] = [output, revisions]
1236
1237         if self.verbose:
1238             print "Label changes: %s" % self.labels.keys()
1239
1240     def guessProjectName(self):
1241         for p in self.depotPaths:
1242             if p.endswith("/"):
1243                 p = p[:-1]
1244             p = p[p.strip().rfind("/") + 1:]
1245             if not p.endswith("/"):
1246                p += "/"
1247             return p
1248
1249     def getBranchMapping(self):
1250         lostAndFoundBranches = set()
1251
1252         for info in p4CmdList("branches"):
1253             details = p4Cmd("branch -o %s" % info["branch"])
1254             viewIdx = 0
1255             while details.has_key("View%s" % viewIdx):
1256                 paths = details["View%s" % viewIdx].split(" ")
1257                 viewIdx = viewIdx + 1
1258                 # require standard //depot/foo/... //depot/bar/... mapping
1259                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1260                     continue
1261                 source = paths[0]
1262                 destination = paths[1]
1263                 ## HACK
1264                 if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]):
1265                     source = source[len(self.depotPaths[0]):-4]
1266                     destination = destination[len(self.depotPaths[0]):-4]
1267
1268                     if destination in self.knownBranches:
1269                         if not self.silent:
1270                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1271                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1272                         continue
1273
1274                     self.knownBranches[destination] = source
1275
1276                     lostAndFoundBranches.discard(destination)
1277
1278                     if source not in self.knownBranches:
1279                         lostAndFoundBranches.add(source)
1280
1281
1282         for branch in lostAndFoundBranches:
1283             self.knownBranches[branch] = branch
1284
1285     def getBranchMappingFromGitBranches(self):
1286         branches = p4BranchesInGit(self.importIntoRemotes)
1287         for branch in branches.keys():
1288             if branch == "master":
1289                 branch = "main"
1290             else:
1291                 branch = branch[len(self.projectName):]
1292             self.knownBranches[branch] = branch
1293
1294     def listExistingP4GitBranches(self):
1295         # branches holds mapping from name to commit
1296         branches = p4BranchesInGit(self.importIntoRemotes)
1297         self.p4BranchesInGit = branches.keys()
1298         for branch in branches.keys():
1299             self.initialParents[self.refPrefix + branch] = branches[branch]
1300
1301     def updateOptionDict(self, d):
1302         option_keys = {}
1303         if self.keepRepoPath:
1304             option_keys['keepRepoPath'] = 1
1305
1306         d["options"] = ' '.join(sorted(option_keys.keys()))
1307
1308     def readOptions(self, d):
1309         self.keepRepoPath = (d.has_key('options')
1310                              and ('keepRepoPath' in d['options']))
1311
1312     def gitRefForBranch(self, branch):
1313         if branch == "main":
1314             return self.refPrefix + "master"
1315
1316         if len(branch) <= 0:
1317             return branch
1318
1319         return self.refPrefix + self.projectName + branch
1320
1321     def gitCommitByP4Change(self, ref, change):
1322         if self.verbose:
1323             print "looking in ref " + ref + " for change %s using bisect..." % change
1324
1325         earliestCommit = ""
1326         latestCommit = parseRevision(ref)
1327
1328         while True:
1329             if self.verbose:
1330                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1331             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1332             if len(next) == 0:
1333                 if self.verbose:
1334                     print "argh"
1335                 return ""
1336             log = extractLogMessageFromGitCommit(next)
1337             settings = extractSettingsGitLog(log)
1338             currentChange = int(settings['change'])
1339             if self.verbose:
1340                 print "current change %s" % currentChange
1341
1342             if currentChange == change:
1343                 if self.verbose:
1344                     print "found %s" % next
1345                 return next
1346
1347             if currentChange < change:
1348                 earliestCommit = "^%s" % next
1349             else:
1350                 latestCommit = "%s" % next
1351
1352         return ""
1353
1354     def importNewBranch(self, branch, maxChange):
1355         # make fast-import flush all changes to disk and update the refs using the checkpoint
1356         # command so that we can try to find the branch parent in the git history
1357         self.gitStream.write("checkpoint\n\n");
1358         self.gitStream.flush();
1359         branchPrefix = self.depotPaths[0] + branch + "/"
1360         range = "@1,%s" % maxChange
1361         #print "prefix" + branchPrefix
1362         changes = p4ChangesForPaths([branchPrefix], range)
1363         if len(changes) <= 0:
1364             return False
1365         firstChange = changes[0]
1366         #print "first change in branch: %s" % firstChange
1367         sourceBranch = self.knownBranches[branch]
1368         sourceDepotPath = self.depotPaths[0] + sourceBranch
1369         sourceRef = self.gitRefForBranch(sourceBranch)
1370         #print "source " + sourceBranch
1371
1372         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1373         #print "branch parent: %s" % branchParentChange
1374         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1375         if len(gitParent) > 0:
1376             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1377             #print "parent git commit: %s" % gitParent
1378
1379         self.importChanges(changes)
1380         return True
1381
1382     def importChanges(self, changes):
1383         cnt = 1
1384         for change in changes:
1385             description = p4Cmd("describe %s" % change)
1386             self.updateOptionDict(description)
1387
1388             if not self.silent:
1389                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1390                 sys.stdout.flush()
1391             cnt = cnt + 1
1392
1393             try:
1394                 if self.detectBranches:
1395                     branches = self.splitFilesIntoBranches(description)
1396                     for branch in branches.keys():
1397                         ## HACK  --hwn
1398                         branchPrefix = self.depotPaths[0] + branch + "/"
1399
1400                         parent = ""
1401
1402                         filesForCommit = branches[branch]
1403
1404                         if self.verbose:
1405                             print "branch is %s" % branch
1406
1407                         self.updatedBranches.add(branch)
1408
1409                         if branch not in self.createdBranches:
1410                             self.createdBranches.add(branch)
1411                             parent = self.knownBranches[branch]
1412                             if parent == branch:
1413                                 parent = ""
1414                             else:
1415                                 fullBranch = self.projectName + branch
1416                                 if fullBranch not in self.p4BranchesInGit:
1417                                     if not self.silent:
1418                                         print("\n    Importing new branch %s" % fullBranch);
1419                                     if self.importNewBranch(branch, change - 1):
1420                                         parent = ""
1421                                         self.p4BranchesInGit.append(fullBranch)
1422                                     if not self.silent:
1423                                         print("\n    Resuming with change %s" % change);
1424
1425                                 if self.verbose:
1426                                     print "parent determined through known branches: %s" % parent
1427
1428                         branch = self.gitRefForBranch(branch)
1429                         parent = self.gitRefForBranch(parent)
1430
1431                         if self.verbose:
1432                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1433
1434                         if len(parent) == 0 and branch in self.initialParents:
1435                             parent = self.initialParents[branch]
1436                             del self.initialParents[branch]
1437
1438                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1439                 else:
1440                     files = self.extractFilesFromCommit(description)
1441                     self.commit(description, files, self.branch, self.depotPaths,
1442                                 self.initialParent)
1443                     self.initialParent = ""
1444             except IOError:
1445                 print self.gitError.read()
1446                 sys.exit(1)
1447
1448     def importHeadRevision(self, revision):
1449         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1450
1451         details = { "user" : "git perforce import user", "time" : int(time.time()) }
1452         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1453                            % (' '.join(self.depotPaths), revision))
1454         details["change"] = revision
1455         newestRevision = 0
1456
1457         fileCnt = 0
1458         for info in p4CmdList("files "
1459                               +  ' '.join(["%s...%s"
1460                                            % (p, revision)
1461                                            for p in self.depotPaths])):
1462
1463             if 'code' in info and info['code'] == 'error':
1464                 sys.stderr.write("p4 returned an error: %s\n"
1465                                  % info['data'])
1466                 if info['data'].find("must refer to client") >= 0:
1467                     sys.stderr.write("This particular p4 error is misleading.\n")
1468                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1469                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1470                 sys.exit(1)
1471             if 'p4ExitCode' in info:
1472                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1473                 sys.exit(1)
1474
1475
1476             change = int(info["change"])
1477             if change > newestRevision:
1478                 newestRevision = change
1479
1480             if info["action"] in self.delete_actions:
1481                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1482                 #fileCnt = fileCnt + 1
1483                 continue
1484
1485             for prop in ["depotFile", "rev", "action", "type" ]:
1486                 details["%s%s" % (prop, fileCnt)] = info[prop]
1487
1488             fileCnt = fileCnt + 1
1489
1490         details["change"] = newestRevision
1491         self.updateOptionDict(details)
1492         try:
1493             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1494         except IOError:
1495             print "IO error with git fast-import. Is your git version recent enough?"
1496             print self.gitError.read()
1497
1498
1499     def getClientSpec(self):
1500         specList = p4CmdList( "client -o" )
1501         temp = {}
1502         for entry in specList:
1503             for k,v in entry.iteritems():
1504                 if k.startswith("View"):
1505                     if v.startswith('"'):
1506                         start = 1
1507                     else:
1508                         start = 0
1509                     index = v.find("...")
1510                     v = v[start:index]
1511                     if v.startswith("-"):
1512                         v = v[1:]
1513                         temp[v] = -len(v)
1514                     else:
1515                         temp[v] = len(v)
1516         self.clientSpecDirs = temp.items()
1517         self.clientSpecDirs.sort( lambda x, y: abs( y[1] ) - abs( x[1] ) )
1518
1519     def run(self, args):
1520         self.depotPaths = []
1521         self.changeRange = ""
1522         self.initialParent = ""
1523         self.previousDepotPaths = []
1524
1525         # map from branch depot path to parent branch
1526         self.knownBranches = {}
1527         self.initialParents = {}
1528         self.hasOrigin = originP4BranchesExist()
1529         if not self.syncWithOrigin:
1530             self.hasOrigin = False
1531
1532         if self.importIntoRemotes:
1533             self.refPrefix = "refs/remotes/p4/"
1534         else:
1535             self.refPrefix = "refs/heads/p4/"
1536
1537         if self.syncWithOrigin and self.hasOrigin:
1538             if not self.silent:
1539                 print "Syncing with origin first by calling git fetch origin"
1540             system("git fetch origin")
1541
1542         if len(self.branch) == 0:
1543             self.branch = self.refPrefix + "master"
1544             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1545                 system("git update-ref %s refs/heads/p4" % self.branch)
1546                 system("git branch -D p4");
1547             # create it /after/ importing, when master exists
1548             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1549                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1550
1551         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1552             self.getClientSpec()
1553
1554         # TODO: should always look at previous commits,
1555         # merge with previous imports, if possible.
1556         if args == []:
1557             if self.hasOrigin:
1558                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1559             self.listExistingP4GitBranches()
1560
1561             if len(self.p4BranchesInGit) > 1:
1562                 if not self.silent:
1563                     print "Importing from/into multiple branches"
1564                 self.detectBranches = True
1565
1566             if self.verbose:
1567                 print "branches: %s" % self.p4BranchesInGit
1568
1569             p4Change = 0
1570             for branch in self.p4BranchesInGit:
1571                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1572
1573                 settings = extractSettingsGitLog(logMsg)
1574
1575                 self.readOptions(settings)
1576                 if (settings.has_key('depot-paths')
1577                     and settings.has_key ('change')):
1578                     change = int(settings['change']) + 1
1579                     p4Change = max(p4Change, change)
1580
1581                     depotPaths = sorted(settings['depot-paths'])
1582                     if self.previousDepotPaths == []:
1583                         self.previousDepotPaths = depotPaths
1584                     else:
1585                         paths = []
1586                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1587                             for i in range(0, min(len(cur), len(prev))):
1588                                 if cur[i] <> prev[i]:
1589                                     i = i - 1
1590                                     break
1591
1592                             paths.append (cur[:i + 1])
1593
1594                         self.previousDepotPaths = paths
1595
1596             if p4Change > 0:
1597                 self.depotPaths = sorted(self.previousDepotPaths)
1598                 self.changeRange = "@%s,#head" % p4Change
1599                 if not self.detectBranches:
1600                     self.initialParent = parseRevision(self.branch)
1601                 if not self.silent and not self.detectBranches:
1602                     print "Performing incremental import into %s git branch" % self.branch
1603
1604         if not self.branch.startswith("refs/"):
1605             self.branch = "refs/heads/" + self.branch
1606
1607         if len(args) == 0 and self.depotPaths:
1608             if not self.silent:
1609                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1610         else:
1611             if self.depotPaths and self.depotPaths != args:
1612                 print ("previous import used depot path %s and now %s was specified. "
1613                        "This doesn't work!" % (' '.join (self.depotPaths),
1614                                                ' '.join (args)))
1615                 sys.exit(1)
1616
1617             self.depotPaths = sorted(args)
1618
1619         revision = ""
1620         self.users = {}
1621
1622         newPaths = []
1623         for p in self.depotPaths:
1624             if p.find("@") != -1:
1625                 atIdx = p.index("@")
1626                 self.changeRange = p[atIdx:]
1627                 if self.changeRange == "@all":
1628                     self.changeRange = ""
1629                 elif ',' not in self.changeRange:
1630                     revision = self.changeRange
1631                     self.changeRange = ""
1632                 p = p[:atIdx]
1633             elif p.find("#") != -1:
1634                 hashIdx = p.index("#")
1635                 revision = p[hashIdx:]
1636                 p = p[:hashIdx]
1637             elif self.previousDepotPaths == []:
1638                 revision = "#head"
1639
1640             p = re.sub ("\.\.\.$", "", p)
1641             if not p.endswith("/"):
1642                 p += "/"
1643
1644             newPaths.append(p)
1645
1646         self.depotPaths = newPaths
1647
1648
1649         self.loadUserMapFromCache()
1650         self.labels = {}
1651         if self.detectLabels:
1652             self.getLabels();
1653
1654         if self.detectBranches:
1655             ## FIXME - what's a P4 projectName ?
1656             self.projectName = self.guessProjectName()
1657
1658             if self.hasOrigin:
1659                 self.getBranchMappingFromGitBranches()
1660             else:
1661                 self.getBranchMapping()
1662             if self.verbose:
1663                 print "p4-git branches: %s" % self.p4BranchesInGit
1664                 print "initial parents: %s" % self.initialParents
1665             for b in self.p4BranchesInGit:
1666                 if b != "master":
1667
1668                     ## FIXME
1669                     b = b[len(self.projectName):]
1670                 self.createdBranches.add(b)
1671
1672         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1673
1674         importProcess = subprocess.Popen(["git", "fast-import"],
1675                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1676                                          stderr=subprocess.PIPE);
1677         self.gitOutput = importProcess.stdout
1678         self.gitStream = importProcess.stdin
1679         self.gitError = importProcess.stderr
1680
1681         if revision:
1682             self.importHeadRevision(revision)
1683         else:
1684             changes = []
1685
1686             if len(self.changesFile) > 0:
1687                 output = open(self.changesFile).readlines()
1688                 changeSet = set()
1689                 for line in output:
1690                     changeSet.add(int(line))
1691
1692                 for change in changeSet:
1693                     changes.append(change)
1694
1695                 changes.sort()
1696             else:
1697                 if not self.p4BranchesInGit:
1698                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
1699                 if self.verbose:
1700                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1701                                                               self.changeRange)
1702                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1703
1704                 if len(self.maxChanges) > 0:
1705                     changes = changes[:min(int(self.maxChanges), len(changes))]
1706
1707             if len(changes) == 0:
1708                 if not self.silent:
1709                     print "No changes to import!"
1710                 return True
1711
1712             if not self.silent and not self.detectBranches:
1713                 print "Import destination: %s" % self.branch
1714
1715             self.updatedBranches = set()
1716
1717             self.importChanges(changes)
1718
1719             if not self.silent:
1720                 print ""
1721                 if len(self.updatedBranches) > 0:
1722                     sys.stdout.write("Updated branches: ")
1723                     for b in self.updatedBranches:
1724                         sys.stdout.write("%s " % b)
1725                     sys.stdout.write("\n")
1726
1727         self.gitStream.close()
1728         if importProcess.wait() != 0:
1729             die("fast-import failed: %s" % self.gitError.read())
1730         self.gitOutput.close()
1731         self.gitError.close()
1732
1733         return True
1734
1735 class P4Rebase(Command):
1736     def __init__(self):
1737         Command.__init__(self)
1738         self.options = [ ]
1739         self.description = ("Fetches the latest revision from perforce and "
1740                             + "rebases the current work (branch) against it")
1741         self.verbose = False
1742
1743     def run(self, args):
1744         sync = P4Sync()
1745         sync.run([])
1746
1747         return self.rebase()
1748
1749     def rebase(self):
1750         if os.system("git update-index --refresh") != 0:
1751             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
1752         if len(read_pipe("git diff-index HEAD --")) > 0:
1753             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
1754
1755         [upstream, settings] = findUpstreamBranchPoint()
1756         if len(upstream) == 0:
1757             die("Cannot find upstream branchpoint for rebase")
1758
1759         # the branchpoint may be p4/foo~3, so strip off the parent
1760         upstream = re.sub("~[0-9]+$", "", upstream)
1761
1762         print "Rebasing the current branch onto %s" % upstream
1763         oldHead = read_pipe("git rev-parse HEAD").strip()
1764         system("git rebase %s" % upstream)
1765         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
1766         return True
1767
1768 class P4Clone(P4Sync):
1769     def __init__(self):
1770         P4Sync.__init__(self)
1771         self.description = "Creates a new git repository and imports from Perforce into it"
1772         self.usage = "usage: %prog [options] //depot/path[@revRange]"
1773         self.options += [
1774             optparse.make_option("--destination", dest="cloneDestination",
1775                                  action='store', default=None,
1776                                  help="where to leave result of the clone"),
1777             optparse.make_option("-/", dest="cloneExclude",
1778                                  action="append", type="string",
1779                                  help="exclude depot path")
1780         ]
1781         self.cloneDestination = None
1782         self.needsGit = False
1783
1784     # This is required for the "append" cloneExclude action
1785     def ensure_value(self, attr, value):
1786         if not hasattr(self, attr) or getattr(self, attr) is None:
1787             setattr(self, attr, value)
1788         return getattr(self, attr)
1789
1790     def defaultDestination(self, args):
1791         ## TODO: use common prefix of args?
1792         depotPath = args[0]
1793         depotDir = re.sub("(@[^@]*)$", "", depotPath)
1794         depotDir = re.sub("(#[^#]*)$", "", depotDir)
1795         depotDir = re.sub(r"\.\.\.$", "", depotDir)
1796         depotDir = re.sub(r"/$", "", depotDir)
1797         return os.path.split(depotDir)[1]
1798
1799     def run(self, args):
1800         if len(args) < 1:
1801             return False
1802
1803         if self.keepRepoPath and not self.cloneDestination:
1804             sys.stderr.write("Must specify destination for --keep-path\n")
1805             sys.exit(1)
1806
1807         depotPaths = args
1808
1809         if not self.cloneDestination and len(depotPaths) > 1:
1810             self.cloneDestination = depotPaths[-1]
1811             depotPaths = depotPaths[:-1]
1812
1813         self.cloneExclude = ["/"+p for p in self.cloneExclude]
1814         for p in depotPaths:
1815             if not p.startswith("//"):
1816                 return False
1817
1818         if not self.cloneDestination:
1819             self.cloneDestination = self.defaultDestination(args)
1820
1821         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
1822         if not os.path.exists(self.cloneDestination):
1823             os.makedirs(self.cloneDestination)
1824         chdir(self.cloneDestination)
1825         system("git init")
1826         self.gitdir = os.getcwd() + "/.git"
1827         if not P4Sync.run(self, depotPaths):
1828             return False
1829         if self.branch != "master":
1830             if self.importIntoRemotes:
1831                 masterbranch = "refs/remotes/p4/master"
1832             else:
1833                 masterbranch = "refs/heads/p4/master"
1834             if gitBranchExists(masterbranch):
1835                 system("git branch master %s" % masterbranch)
1836                 system("git checkout -f")
1837             else:
1838                 print "Could not detect main branch. No checkout/master branch created."
1839
1840         return True
1841
1842 class P4Branches(Command):
1843     def __init__(self):
1844         Command.__init__(self)
1845         self.options = [ ]
1846         self.description = ("Shows the git branches that hold imports and their "
1847                             + "corresponding perforce depot paths")
1848         self.verbose = False
1849
1850     def run(self, args):
1851         if originP4BranchesExist():
1852             createOrUpdateBranchesFromOrigin()
1853
1854         cmdline = "git rev-parse --symbolic "
1855         cmdline += " --remotes"
1856
1857         for line in read_pipe_lines(cmdline):
1858             line = line.strip()
1859
1860             if not line.startswith('p4/') or line == "p4/HEAD":
1861                 continue
1862             branch = line
1863
1864             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
1865             settings = extractSettingsGitLog(log)
1866
1867             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
1868         return True
1869
1870 class HelpFormatter(optparse.IndentedHelpFormatter):
1871     def __init__(self):
1872         optparse.IndentedHelpFormatter.__init__(self)
1873
1874     def format_description(self, description):
1875         if description:
1876             return description + "\n"
1877         else:
1878             return ""
1879
1880 def printUsage(commands):
1881     print "usage: %s <command> [options]" % sys.argv[0]
1882     print ""
1883     print "valid commands: %s" % ", ".join(commands)
1884     print ""
1885     print "Try %s <command> --help for command specific help." % sys.argv[0]
1886     print ""
1887
1888 commands = {
1889     "debug" : P4Debug,
1890     "submit" : P4Submit,
1891     "commit" : P4Submit,
1892     "sync" : P4Sync,
1893     "rebase" : P4Rebase,
1894     "clone" : P4Clone,
1895     "rollback" : P4RollBack,
1896     "branches" : P4Branches
1897 }
1898
1899
1900 def main():
1901     if len(sys.argv[1:]) == 0:
1902         printUsage(commands.keys())
1903         sys.exit(2)
1904
1905     cmd = ""
1906     cmdName = sys.argv[1]
1907     try:
1908         klass = commands[cmdName]
1909         cmd = klass()
1910     except KeyError:
1911         print "unknown command %s" % cmdName
1912         print ""
1913         printUsage(commands.keys())
1914         sys.exit(2)
1915
1916     options = cmd.options
1917     cmd.gitdir = os.environ.get("GIT_DIR", None)
1918
1919     args = sys.argv[2:]
1920
1921     if len(options) > 0:
1922         options.append(optparse.make_option("--git-dir", dest="gitdir"))
1923
1924         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1925                                        options,
1926                                        description = cmd.description,
1927                                        formatter = HelpFormatter())
1928
1929         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1930     global verbose
1931     verbose = cmd.verbose
1932     if cmd.needsGit:
1933         if cmd.gitdir == None:
1934             cmd.gitdir = os.path.abspath(".git")
1935             if not isValidGitDir(cmd.gitdir):
1936                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
1937                 if os.path.exists(cmd.gitdir):
1938                     cdup = read_pipe("git rev-parse --show-cdup").strip()
1939                     if len(cdup) > 0:
1940                         chdir(cdup);
1941
1942         if not isValidGitDir(cmd.gitdir):
1943             if isValidGitDir(cmd.gitdir + "/.git"):
1944                 cmd.gitdir += "/.git"
1945             else:
1946                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
1947
1948         os.environ["GIT_DIR"] = cmd.gitdir
1949
1950     if not cmd.run(args):
1951         parser.print_help()
1952
1953
1954 if __name__ == '__main__':
1955     main()