contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = ["p4"]
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += ["-u",user]
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += ["-P", password]
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += ["-p", port]
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += ["-h", host]
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += ["-c", client]
  46
  47
  48     if isinstance(cmd,basestring):
  49         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50     else:
  51         real_cmd += cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56     # not using the shell, we have to set it ourselves.
  57     os.environ['PWD']=dir
  58     os.chdir(dir)
  59
  60 def die(msg):
  61     if verbose:
  62         raise Exception(msg)
  63     else:
  64         sys.stderr.write(msg + "\n")
  65         sys.exit(1)
  66
  67 def write_pipe(c, stdin):
  68     if verbose:
  69         sys.stderr.write('Writing pipe: %s\n' % str(c))
  70
  71     expand = isinstance(c,basestring)
  72     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  73     pipe = p.stdin
  74     val = pipe.write(stdin)
  75     pipe.close()
  76     if p.wait():
  77         die('Command failed: %s' % str(c))
  78
  79     return val
  80
  81 def p4_write_pipe(c, stdin):
  82     real_cmd = p4_build_cmd(c)
  83     return write_pipe(real_cmd, stdin)
  84
  85 def read_pipe(c, ignore_error=False):
  86     if verbose:
  87         sys.stderr.write('Reading pipe: %s\n' % str(c))
  88
  89     expand = isinstance(c,basestring)
  90     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  91     pipe = p.stdout
  92     val = pipe.read()
  93     if p.wait() and not ignore_error:
  94         die('Command failed: %s' % str(c))
  95
  96     return val
  97
  98 def p4_read_pipe(c, ignore_error=False):
  99     real_cmd = p4_build_cmd(c)
 100     return read_pipe(real_cmd, ignore_error)
 101
 102 def read_pipe_lines(c):
 103     if verbose:
 104         sys.stderr.write('Reading pipe: %s\n' % str(c))
 105
 106     expand = isinstance(c, basestring)
 107     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 108     pipe = p.stdout
 109     val = pipe.readlines()
 110     if pipe.close() or p.wait():
 111         die('Command failed: %s' % str(c))
 112
 113     return val
 114
 115 def p4_read_pipe_lines(c):
 116     """Specifically invoke p4 on the command supplied. """
 117     real_cmd = p4_build_cmd(c)
 118     return read_pipe_lines(real_cmd)
 119
 120 def system(cmd):
 121     expand = isinstance(cmd,basestring)
 122     if verbose:
 123         sys.stderr.write("executing %s\n" % str(cmd))
 124     subprocess.check_call(cmd, shell=expand)
 125
 126 def p4_system(cmd):
 127     """Specifically invoke p4 as the system command. """
 128     real_cmd = p4_build_cmd(cmd)
 129     expand = isinstance(real_cmd, basestring)
 130     subprocess.check_call(real_cmd, shell=expand)
 131
 132 def p4_integrate(src, dest):
 133     p4_system(["integrate", "-Dt", src, dest])
 134
 135 def p4_sync(path):
 136     p4_system(["sync", path])
 137
 138 def p4_add(f):
 139     p4_system(["add", f])
 140
 141 def p4_delete(f):
 142     p4_system(["delete", f])
 143
 144 def p4_edit(f):
 145     p4_system(["edit", f])
 146
 147 def p4_revert(f):
 148     p4_system(["revert", f])
 149
 150 def p4_reopen(type, file):
 151     p4_system(["reopen", "-t", type, file])
 152
 153 #
 154 # Canonicalize the p4 type and return a tuple of the
 155 # base type, plus any modifiers.  See "p4 help filetypes"
 156 # for a list and explanation.
 157 #
 158 def split_p4_type(p4type):
 159
 160     p4_filetypes_historical = {
 161         "ctempobj": "binary+Sw",
 162         "ctext": "text+C",
 163         "cxtext": "text+Cx",
 164         "ktext": "text+k",
 165         "kxtext": "text+kx",
 166         "ltext": "text+F",
 167         "tempobj": "binary+FSw",
 168         "ubinary": "binary+F",
 169         "uresource": "resource+F",
 170         "uxbinary": "binary+Fx",
 171         "xbinary": "binary+x",
 172         "xltext": "text+Fx",
 173         "xtempobj": "binary+Swx",
 174         "xtext": "text+x",
 175         "xunicode": "unicode+x",
 176         "xutf16": "utf16+x",
 177     }
 178     if p4type in p4_filetypes_historical:
 179         p4type = p4_filetypes_historical[p4type]
 180     mods = ""
 181     s = p4type.split("+")
 182     base = s[0]
 183     mods = ""
 184     if len(s) > 1:
 185         mods = s[1]
 186     return (base, mods)
 187
 188
 189 def setP4ExecBit(file, mode):
 190     # Reopens an already open file and changes the execute bit to match
 191     # the execute bit setting in the passed in mode.
 192
 193     p4Type = "+x"
 194
 195     if not isModeExec(mode):
 196         p4Type = getP4OpenedType(file)
 197         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 198         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 199         if p4Type[-1] == "+":
 200             p4Type = p4Type[0:-1]
 201
 202     p4_reopen(p4Type, file)
 203
 204 def getP4OpenedType(file):
 205     # Returns the perforce file type for the given file.
 206
 207     result = p4_read_pipe(["opened", file])
 208     match = re.match(".*\((.+)\)\r?$", result)
 209     if match:
 210         return match.group(1)
 211     else:
 212         die("Could not determine file type for %s (result: '%s')" % (file, result))
 213
 214 def diffTreePattern():
 215     # This is a simple generator for the diff tree regex pattern. This could be
 216     # a class variable if this and parseDiffTreeEntry were a part of a class.
 217     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 218     while True:
 219         yield pattern
 220
 221 def parseDiffTreeEntry(entry):
 222     """Parses a single diff tree entry into its component elements.
 223
 224     See git-diff-tree(1) manpage for details about the format of the diff
 225     output. This method returns a dictionary with the following elements:
 226
 227     src_mode - The mode of the source file
 228     dst_mode - The mode of the destination file
 229     src_sha1 - The sha1 for the source file
 230     dst_sha1 - The sha1 fr the destination file
 231     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 232     status_score - The score for the status (applicable for 'C' and 'R'
 233                    statuses). This is None if there is no score.
 234     src - The path for the source file.
 235     dst - The path for the destination file. This is only present for
 236           copy or renames. If it is not present, this is None.
 237
 238     If the pattern is not matched, None is returned."""
 239
 240     match = diffTreePattern().next().match(entry)
 241     if match:
 242         return {
 243             'src_mode': match.group(1),
 244             'dst_mode': match.group(2),
 245             'src_sha1': match.group(3),
 246             'dst_sha1': match.group(4),
 247             'status': match.group(5),
 248             'status_score': match.group(6),
 249             'src': match.group(7),
 250             'dst': match.group(10)
 251         }
 252     return None
 253
 254 def isModeExec(mode):
 255     # Returns True if the given git mode represents an executable file,
 256     # otherwise False.
 257     return mode[-3:] == "755"
 258
 259 def isModeExecChanged(src_mode, dst_mode):
 260     return isModeExec(src_mode) != isModeExec(dst_mode)
 261
 262 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 263
 264     if isinstance(cmd,basestring):
 265         cmd = "-G " + cmd
 266         expand = True
 267     else:
 268         cmd = ["-G"] + cmd
 269         expand = False
 270
 271     cmd = p4_build_cmd(cmd)
 272     if verbose:
 273         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 274
 275     # Use a temporary file to avoid deadlocks without
 276     # subprocess.communicate(), which would put another copy
 277     # of stdout into memory.
 278     stdin_file = None
 279     if stdin is not None:
 280         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 281         if isinstance(stdin,basestring):
 282             stdin_file.write(stdin)
 283         else:
 284             for i in stdin:
 285                 stdin_file.write(i + '\n')
 286         stdin_file.flush()
 287         stdin_file.seek(0)
 288
 289     p4 = subprocess.Popen(cmd,
 290                           shell=expand,
 291                           stdin=stdin_file,
 292                           stdout=subprocess.PIPE)
 293
 294     result = []
 295     try:
 296         while True:
 297             entry = marshal.load(p4.stdout)
 298             if cb is not None:
 299                 cb(entry)
 300             else:
 301                 result.append(entry)
 302     except EOFError:
 303         pass
 304     exitCode = p4.wait()
 305     if exitCode != 0:
 306         entry = {}
 307         entry["p4ExitCode"] = exitCode
 308         result.append(entry)
 309
 310     return result
 311
 312 def p4Cmd(cmd):
 313     list = p4CmdList(cmd)
 314     result = {}
 315     for entry in list:
 316         result.update(entry)
 317     return result;
 318
 319 def p4Where(depotPath):
 320     if not depotPath.endswith("/"):
 321         depotPath += "/"
 322     depotPath = depotPath + "..."
 323     outputList = p4CmdList(["where", depotPath])
 324     output = None
 325     for entry in outputList:
 326         if "depotFile" in entry:
 327             if entry["depotFile"] == depotPath:
 328                 output = entry
 329                 break
 330         elif "data" in entry:
 331             data = entry.get("data")
 332             space = data.find(" ")
 333             if data[:space] == depotPath:
 334                 output = entry
 335                 break
 336     if output == None:
 337         return ""
 338     if output["code"] == "error":
 339         return ""
 340     clientPath = ""
 341     if "path" in output:
 342         clientPath = output.get("path")
 343     elif "data" in output:
 344         data = output.get("data")
 345         lastSpace = data.rfind(" ")
 346         clientPath = data[lastSpace + 1:]
 347
 348     if clientPath.endswith("..."):
 349         clientPath = clientPath[:-3]
 350     return clientPath
 351
 352 def currentGitBranch():
 353     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 354
 355 def isValidGitDir(path):
 356     if (os.path.exists(path + "/HEAD")
 357         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 358         return True;
 359     return False
 360
 361 def parseRevision(ref):
 362     return read_pipe("git rev-parse %s" % ref).strip()
 363
 364 def extractLogMessageFromGitCommit(commit):
 365     logMessage = ""
 366
 367     ## fixme: title is first line of commit, not 1st paragraph.
 368     foundTitle = False
 369     for log in read_pipe_lines("git cat-file commit %s" % commit):
 370        if not foundTitle:
 371            if len(log) == 1:
 372                foundTitle = True
 373            continue
 374
 375        logMessage += log
 376     return logMessage
 377
 378 def extractSettingsGitLog(log):
 379     values = {}
 380     for line in log.split("\n"):
 381         line = line.strip()
 382         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 383         if not m:
 384             continue
 385
 386         assignments = m.group(1).split (':')
 387         for a in assignments:
 388             vals = a.split ('=')
 389             key = vals[0].strip()
 390             val = ('='.join (vals[1:])).strip()
 391             if val.endswith ('\"') and val.startswith('"'):
 392                 val = val[1:-1]
 393
 394             values[key] = val
 395
 396     paths = values.get("depot-paths")
 397     if not paths:
 398         paths = values.get("depot-path")
 399     if paths:
 400         values['depot-paths'] = paths.split(',')
 401     return values
 402
 403 def gitBranchExists(branch):
 404     proc = subprocess.Popen(["git", "rev-parse", branch],
 405                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 406     return proc.wait() == 0;
 407
 408 _gitConfig = {}
 409 def gitConfig(key, args = None): # set args to "--bool", for instance
 410     if not _gitConfig.has_key(key):
 411         argsFilter = ""
 412         if args != None:
 413             argsFilter = "%s " % args
 414         cmd = "git config %s%s" % (argsFilter, key)
 415         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 416     return _gitConfig[key]
 417
 418 def gitConfigList(key):
 419     if not _gitConfig.has_key(key):
 420         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 421     return _gitConfig[key]
 422
 423 def p4BranchesInGit(branchesAreInRemotes = True):
 424     branches = {}
 425
 426     cmdline = "git rev-parse --symbolic "
 427     if branchesAreInRemotes:
 428         cmdline += " --remotes"
 429     else:
 430         cmdline += " --branches"
 431
 432     for line in read_pipe_lines(cmdline):
 433         line = line.strip()
 434
 435         ## only import to p4/
 436         if not line.startswith('p4/') or line == "p4/HEAD":
 437             continue
 438         branch = line
 439
 440         # strip off p4
 441         branch = re.sub ("^p4/", "", line)
 442
 443         branches[branch] = parseRevision(line)
 444     return branches
 445
 446 def findUpstreamBranchPoint(head = "HEAD"):
 447     branches = p4BranchesInGit()
 448     # map from depot-path to branch name
 449     branchByDepotPath = {}
 450     for branch in branches.keys():
 451         tip = branches[branch]
 452         log = extractLogMessageFromGitCommit(tip)
 453         settings = extractSettingsGitLog(log)
 454         if settings.has_key("depot-paths"):
 455             paths = ",".join(settings["depot-paths"])
 456             branchByDepotPath[paths] = "remotes/p4/" + branch
 457
 458     settings = None
 459     parent = 0
 460     while parent < 65535:
 461         commit = head + "~%s" % parent
 462         log = extractLogMessageFromGitCommit(commit)
 463         settings = extractSettingsGitLog(log)
 464         if settings.has_key("depot-paths"):
 465             paths = ",".join(settings["depot-paths"])
 466             if branchByDepotPath.has_key(paths):
 467                 return [branchByDepotPath[paths], settings]
 468
 469         parent = parent + 1
 470
 471     return ["", settings]
 472
 473 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 474     if not silent:
 475         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 476                % localRefPrefix)
 477
 478     originPrefix = "origin/p4/"
 479
 480     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 481         line = line.strip()
 482         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 483             continue
 484
 485         headName = line[len(originPrefix):]
 486         remoteHead = localRefPrefix + headName
 487         originHead = line
 488
 489         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 490         if (not original.has_key('depot-paths')
 491             or not original.has_key('change')):
 492             continue
 493
 494         update = False
 495         if not gitBranchExists(remoteHead):
 496             if verbose:
 497                 print "creating %s" % remoteHead
 498             update = True
 499         else:
 500             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 501             if settings.has_key('change') > 0:
 502                 if settings['depot-paths'] == original['depot-paths']:
 503                     originP4Change = int(original['change'])
 504                     p4Change = int(settings['change'])
 505                     if originP4Change > p4Change:
 506                         print ("%s (%s) is newer than %s (%s). "
 507                                "Updating p4 branch from origin."
 508                                % (originHead, originP4Change,
 509                                   remoteHead, p4Change))
 510                         update = True
 511                 else:
 512                     print ("Ignoring: %s was imported from %s while "
 513                            "%s was imported from %s"
 514                            % (originHead, ','.join(original['depot-paths']),
 515                               remoteHead, ','.join(settings['depot-paths'])))
 516
 517         if update:
 518             system("git update-ref %s %s" % (remoteHead, originHead))
 519
 520 def originP4BranchesExist():
 521         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 522
 523 def p4ChangesForPaths(depotPaths, changeRange):
 524     assert depotPaths
 525     cmd = ['changes']
 526     for p in depotPaths:
 527         cmd += ["%s...%s" % (p, changeRange)]
 528     output = p4_read_pipe_lines(cmd)
 529
 530     changes = {}
 531     for line in output:
 532         changeNum = int(line.split(" ")[1])
 533         changes[changeNum] = True
 534
 535     changelist = changes.keys()
 536     changelist.sort()
 537     return changelist
 538
 539 def p4PathStartsWith(path, prefix):
 540     # This method tries to remedy a potential mixed-case issue:
 541     #
 542     # If UserA adds  //depot/DirA/file1
 543     # and UserB adds //depot/dira/file2
 544     #
 545     # we may or may not have a problem. If you have core.ignorecase=true,
 546     # we treat DirA and dira as the same directory
 547     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 548     if ignorecase:
 549         return path.lower().startswith(prefix.lower())
 550     return path.startswith(prefix)
 551
 552 class Command:
 553     def __init__(self):
 554         self.usage = "usage: %prog [options]"
 555         self.needsGit = True
 556
 557 class P4UserMap:
 558     def __init__(self):
 559         self.userMapFromPerforceServer = False
 560
 561     def getUserCacheFilename(self):
 562         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 563         return home + "/.gitp4-usercache.txt"
 564
 565     def getUserMapFromPerforceServer(self):
 566         if self.userMapFromPerforceServer:
 567             return
 568         self.users = {}
 569         self.emails = {}
 570
 571         for output in p4CmdList("users"):
 572             if not output.has_key("User"):
 573                 continue
 574             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 575             self.emails[output["Email"]] = output["User"]
 576
 577
 578         s = ''
 579         for (key, val) in self.users.items():
 580             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 581
 582         open(self.getUserCacheFilename(), "wb").write(s)
 583         self.userMapFromPerforceServer = True
 584
 585     def loadUserMapFromCache(self):
 586         self.users = {}
 587         self.userMapFromPerforceServer = False
 588         try:
 589             cache = open(self.getUserCacheFilename(), "rb")
 590             lines = cache.readlines()
 591             cache.close()
 592             for line in lines:
 593                 entry = line.strip().split("\t")
 594                 self.users[entry[0]] = entry[1]
 595         except IOError:
 596             self.getUserMapFromPerforceServer()
 597
 598 class P4Debug(Command):
 599     def __init__(self):
 600         Command.__init__(self)
 601         self.options = [
 602             optparse.make_option("--verbose", dest="verbose", action="store_true",
 603                                  default=False),
 604             ]
 605         self.description = "A tool to debug the output of p4 -G."
 606         self.needsGit = False
 607         self.verbose = False
 608
 609     def run(self, args):
 610         j = 0
 611         for output in p4CmdList(args):
 612             print 'Element: %d' % j
 613             j += 1
 614             print output
 615         return True
 616
 617 class P4RollBack(Command):
 618     def __init__(self):
 619         Command.__init__(self)
 620         self.options = [
 621             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 622             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 623         ]
 624         self.description = "A tool to debug the multi-branch import. Don't use :)"
 625         self.verbose = False
 626         self.rollbackLocalBranches = False
 627
 628     def run(self, args):
 629         if len(args) != 1:
 630             return False
 631         maxChange = int(args[0])
 632
 633         if "p4ExitCode" in p4Cmd("changes -m 1"):
 634             die("Problems executing p4");
 635
 636         if self.rollbackLocalBranches:
 637             refPrefix = "refs/heads/"
 638             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 639         else:
 640             refPrefix = "refs/remotes/"
 641             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 642
 643         for line in lines:
 644             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 645                 line = line.strip()
 646                 ref = refPrefix + line
 647                 log = extractLogMessageFromGitCommit(ref)
 648                 settings = extractSettingsGitLog(log)
 649
 650                 depotPaths = settings['depot-paths']
 651                 change = settings['change']
 652
 653                 changed = False
 654
 655                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 656                                                            for p in depotPaths]))) == 0:
 657                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 658                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 659                     continue
 660
 661                 while change and int(change) > maxChange:
 662                     changed = True
 663                     if self.verbose:
 664                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 665                     system("git update-ref %s \"%s^\"" % (ref, ref))
 666                     log = extractLogMessageFromGitCommit(ref)
 667                     settings =  extractSettingsGitLog(log)
 668
 669
 670                     depotPaths = settings['depot-paths']
 671                     change = settings['change']
 672
 673                 if changed:
 674                     print "%s rewound to %s" % (ref, change)
 675
 676         return True
 677
 678 class P4Submit(Command, P4UserMap):
 679     def __init__(self):
 680         Command.__init__(self)
 681         P4UserMap.__init__(self)
 682         self.options = [
 683                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 684                 optparse.make_option("--origin", dest="origin"),
 685                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 686                 # preserve the user, requires relevant p4 permissions
 687                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 688         ]
 689         self.description = "Submit changes from git to the perforce depot."
 690         self.usage += " [name of git branch to submit into perforce depot]"
 691         self.interactive = True
 692         self.origin = ""
 693         self.detectRenames = False
 694         self.verbose = False
 695         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 696         self.isWindows = (platform.system() == "Windows")
 697         self.myP4UserId = None
 698
 699     def check(self):
 700         if len(p4CmdList("opened ...")) > 0:
 701             die("You have files opened with perforce! Close them before starting the sync.")
 702
 703     # replaces everything between 'Description:' and the next P4 submit template field with the
 704     # commit message
 705     def prepareLogMessage(self, template, message):
 706         result = ""
 707
 708         inDescriptionSection = False
 709
 710         for line in template.split("\n"):
 711             if line.startswith("#"):
 712                 result += line + "\n"
 713                 continue
 714
 715             if inDescriptionSection:
 716                 if line.startswith("Files:") or line.startswith("Jobs:"):
 717                     inDescriptionSection = False
 718                 else:
 719                     continue
 720             else:
 721                 if line.startswith("Description:"):
 722                     inDescriptionSection = True
 723                     line += "\n"
 724                     for messageLine in message.split("\n"):
 725                         line += "\t" + messageLine + "\n"
 726
 727             result += line + "\n"
 728
 729         return result
 730
 731     def p4UserForCommit(self,id):
 732         # Return the tuple (perforce user,git email) for a given git commit id
 733         self.getUserMapFromPerforceServer()
 734         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 735         gitEmail = gitEmail.strip()
 736         if not self.emails.has_key(gitEmail):
 737             return (None,gitEmail)
 738         else:
 739             return (self.emails[gitEmail],gitEmail)
 740
 741     def checkValidP4Users(self,commits):
 742         # check if any git authors cannot be mapped to p4 users
 743         for id in commits:
 744             (user,email) = self.p4UserForCommit(id)
 745             if not user:
 746                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 747                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 748                     print "%s" % msg
 749                 else:
 750                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 751
 752     def lastP4Changelist(self):
 753         # Get back the last changelist number submitted in this client spec. This
 754         # then gets used to patch up the username in the change. If the same
 755         # client spec is being used by multiple processes then this might go
 756         # wrong.
 757         results = p4CmdList("client -o")        # find the current client
 758         client = None
 759         for r in results:
 760             if r.has_key('Client'):
 761                 client = r['Client']
 762                 break
 763         if not client:
 764             die("could not get client spec")
 765         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 766         for r in results:
 767             if r.has_key('change'):
 768                 return r['change']
 769         die("Could not get changelist number for last submit - cannot patch up user details")
 770
 771     def modifyChangelistUser(self, changelist, newUser):
 772         # fixup the user field of a changelist after it has been submitted.
 773         changes = p4CmdList("change -o %s" % changelist)
 774         if len(changes) != 1:
 775             die("Bad output from p4 change modifying %s to user %s" %
 776                 (changelist, newUser))
 777
 778         c = changes[0]
 779         if c['User'] == newUser: return   # nothing to do
 780         c['User'] = newUser
 781         input = marshal.dumps(c)
 782
 783         result = p4CmdList("change -f -i", stdin=input)
 784         for r in result:
 785             if r.has_key('code'):
 786                 if r['code'] == 'error':
 787                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 788             if r.has_key('data'):
 789                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 790                 return
 791         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 792
 793     def canChangeChangelists(self):
 794         # check to see if we have p4 admin or super-user permissions, either of
 795         # which are required to modify changelists.
 796         results = p4CmdList("protects %s" % self.depotPath)
 797         for r in results:
 798             if r.has_key('perm'):
 799                 if r['perm'] == 'admin':
 800                     return 1
 801                 if r['perm'] == 'super':
 802                     return 1
 803         return 0
 804
 805     def p4UserId(self):
 806         if self.myP4UserId:
 807             return self.myP4UserId
 808
 809         results = p4CmdList("user -o")
 810         for r in results:
 811             if r.has_key('User'):
 812                 self.myP4UserId = r['User']
 813                 return r['User']
 814         die("Could not find your p4 user id")
 815
 816     def p4UserIsMe(self, p4User):
 817         # return True if the given p4 user is actually me
 818         me = self.p4UserId()
 819         if not p4User or p4User != me:
 820             return False
 821         else:
 822             return True
 823
 824     def prepareSubmitTemplate(self):
 825         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 826         template = ""
 827         inFilesSection = False
 828         for line in p4_read_pipe_lines(['change', '-o']):
 829             if line.endswith("\r\n"):
 830                 line = line[:-2] + "\n"
 831             if inFilesSection:
 832                 if line.startswith("\t"):
 833                     # path starts and ends with a tab
 834                     path = line[1:]
 835                     lastTab = path.rfind("\t")
 836                     if lastTab != -1:
 837                         path = path[:lastTab]
 838                         if not p4PathStartsWith(path, self.depotPath):
 839                             continue
 840                 else:
 841                     inFilesSection = False
 842             else:
 843                 if line.startswith("Files:"):
 844                     inFilesSection = True
 845
 846             template += line
 847
 848         return template
 849
 850     def edit_template(self, template_file):
 851         """Invoke the editor to let the user change the submission
 852            message.  Return true if okay to continue with the submit."""
 853
 854         # if configured to skip the editing part, just submit
 855         if gitConfig("git-p4.skipSubmitEdit") == "true":
 856             return True
 857
 858         # look at the modification time, to check later if the user saved
 859         # the file
 860         mtime = os.stat(template_file).st_mtime
 861
 862         # invoke the editor
 863         if os.environ.has_key("P4EDITOR"):
 864             editor = os.environ.get("P4EDITOR")
 865         else:
 866             editor = read_pipe("git var GIT_EDITOR").strip()
 867         system(editor + " " + template_file)
 868
 869         # If the file was not saved, prompt to see if this patch should
 870         # be skipped.  But skip this verification step if configured so.
 871         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 872             return True
 873
 874         if os.stat(template_file).st_mtime <= mtime:
 875             while True:
 876                 response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 877                 if response == 'y':
 878                     return True
 879                 if response == 'n':
 880                     return False
 881
 882     def applyCommit(self, id):
 883         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 884
 885         (p4User, gitEmail) = self.p4UserForCommit(id)
 886
 887         if not self.detectRenames:
 888             # If not explicitly set check the config variable
 889             self.detectRenames = gitConfig("git-p4.detectRenames")
 890
 891         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 892             diffOpts = ""
 893         elif self.detectRenames.lower() == "true":
 894             diffOpts = "-M"
 895         else:
 896             diffOpts = "-M%s" % self.detectRenames
 897
 898         detectCopies = gitConfig("git-p4.detectCopies")
 899         if detectCopies.lower() == "true":
 900             diffOpts += " -C"
 901         elif detectCopies != "" and detectCopies.lower() != "false":
 902             diffOpts += " -C%s" % detectCopies
 903
 904         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 905             diffOpts += " --find-copies-harder"
 906
 907         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 908         filesToAdd = set()
 909         filesToDelete = set()
 910         editedFiles = set()
 911         filesToChangeExecBit = {}
 912         for line in diff:
 913             diff = parseDiffTreeEntry(line)
 914             modifier = diff['status']
 915             path = diff['src']
 916             if modifier == "M":
 917                 p4_edit(path)
 918                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 919                     filesToChangeExecBit[path] = diff['dst_mode']
 920                 editedFiles.add(path)
 921             elif modifier == "A":
 922                 filesToAdd.add(path)
 923                 filesToChangeExecBit[path] = diff['dst_mode']
 924                 if path in filesToDelete:
 925                     filesToDelete.remove(path)
 926             elif modifier == "D":
 927                 filesToDelete.add(path)
 928                 if path in filesToAdd:
 929                     filesToAdd.remove(path)
 930             elif modifier == "C":
 931                 src, dest = diff['src'], diff['dst']
 932                 p4_integrate(src, dest)
 933                 if diff['src_sha1'] != diff['dst_sha1']:
 934                     p4_edit(dest)
 935                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 936                     p4_edit(dest)
 937                     filesToChangeExecBit[dest] = diff['dst_mode']
 938                 os.unlink(dest)
 939                 editedFiles.add(dest)
 940             elif modifier == "R":
 941                 src, dest = diff['src'], diff['dst']
 942                 p4_integrate(src, dest)
 943                 if diff['src_sha1'] != diff['dst_sha1']:
 944                     p4_edit(dest)
 945                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 946                     p4_edit(dest)
 947                     filesToChangeExecBit[dest] = diff['dst_mode']
 948                 os.unlink(dest)
 949                 editedFiles.add(dest)
 950                 filesToDelete.add(src)
 951             else:
 952                 die("unknown modifier %s for %s" % (modifier, path))
 953
 954         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 955         patchcmd = diffcmd + " | git apply "
 956         tryPatchCmd = patchcmd + "--check -"
 957         applyPatchCmd = patchcmd + "--check --apply -"
 958
 959         if os.system(tryPatchCmd) != 0:
 960             print "Unfortunately applying the change failed!"
 961             print "What do you want to do?"
 962             response = "x"
 963             while response != "s" and response != "a" and response != "w":
 964                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 965                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 966             if response == "s":
 967                 print "Skipping! Good luck with the next patches..."
 968                 for f in editedFiles:
 969                     p4_revert(f)
 970                 for f in filesToAdd:
 971                     os.remove(f)
 972                 return
 973             elif response == "a":
 974                 os.system(applyPatchCmd)
 975                 if len(filesToAdd) > 0:
 976                     print "You may also want to call p4 add on the following files:"
 977                     print " ".join(filesToAdd)
 978                 if len(filesToDelete):
 979                     print "The following files should be scheduled for deletion with p4 delete:"
 980                     print " ".join(filesToDelete)
 981                 die("Please resolve and submit the conflict manually and "
 982                     + "continue afterwards with git-p4 submit --continue")
 983             elif response == "w":
 984                 system(diffcmd + " > patch.txt")
 985                 print "Patch saved to patch.txt in %s !" % self.clientPath
 986                 die("Please resolve and submit the conflict manually and "
 987                     "continue afterwards with git-p4 submit --continue")
 988
 989         system(applyPatchCmd)
 990
 991         for f in filesToAdd:
 992             p4_add(f)
 993         for f in filesToDelete:
 994             p4_revert(f)
 995             p4_delete(f)
 996
 997         # Set/clear executable bits
 998         for f in filesToChangeExecBit.keys():
 999             mode = filesToChangeExecBit[f]
1000             setP4ExecBit(f, mode)
1001
1002         logMessage = extractLogMessageFromGitCommit(id)
1003         logMessage = logMessage.strip()
1004
1005         template = self.prepareSubmitTemplate()
1006
1007         if self.interactive:
1008             submitTemplate = self.prepareLogMessage(template, logMessage)
1009
1010             if self.preserveUser:
1011                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1012
1013             if os.environ.has_key("P4DIFF"):
1014                 del(os.environ["P4DIFF"])
1015             diff = ""
1016             for editedFile in editedFiles:
1017                 diff += p4_read_pipe(['diff', '-du', editedFile])
1018
1019             newdiff = ""
1020             for newFile in filesToAdd:
1021                 newdiff += "==== new file ====\n"
1022                 newdiff += "--- /dev/null\n"
1023                 newdiff += "+++ %s\n" % newFile
1024                 f = open(newFile, "r")
1025                 for line in f.readlines():
1026                     newdiff += "+" + line
1027                 f.close()
1028
1029             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1030                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1031                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1032                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1033
1034             separatorLine = "######## everything below this line is just the diff #######\n"
1035
1036             (handle, fileName) = tempfile.mkstemp()
1037             tmpFile = os.fdopen(handle, "w+")
1038             if self.isWindows:
1039                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1040                 separatorLine = separatorLine.replace("\n", "\r\n")
1041                 newdiff = newdiff.replace("\n", "\r\n")
1042             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1043             tmpFile.close()
1044
1045             if self.edit_template(fileName):
1046                 # read the edited message and submit
1047                 tmpFile = open(fileName, "rb")
1048                 message = tmpFile.read()
1049                 tmpFile.close()
1050                 submitTemplate = message[:message.index(separatorLine)]
1051                 if self.isWindows:
1052                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1053                 p4_write_pipe(['submit', '-i'], submitTemplate)
1054
1055                 if self.preserveUser:
1056                     if p4User:
1057                         # Get last changelist number. Cannot easily get it from
1058                         # the submit command output as the output is
1059                         # unmarshalled.
1060                         changelist = self.lastP4Changelist()
1061                         self.modifyChangelistUser(changelist, p4User)
1062             else:
1063                 # skip this patch
1064                 for f in editedFiles:
1065                     p4_revert(f)
1066                 for f in filesToAdd:
1067                     p4_revert(f)
1068                     os.remove(f)
1069
1070             os.remove(fileName)
1071         else:
1072             fileName = "submit.txt"
1073             file = open(fileName, "w+")
1074             file.write(self.prepareLogMessage(template, logMessage))
1075             file.close()
1076             print ("Perforce submit template written as %s. "
1077                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1078                    % (fileName, fileName))
1079
1080     def run(self, args):
1081         if len(args) == 0:
1082             self.master = currentGitBranch()
1083             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1084                 die("Detecting current git branch failed!")
1085         elif len(args) == 1:
1086             self.master = args[0]
1087         else:
1088             return False
1089
1090         allowSubmit = gitConfig("git-p4.allowSubmit")
1091         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1092             die("%s is not in git-p4.allowSubmit" % self.master)
1093
1094         [upstream, settings] = findUpstreamBranchPoint()
1095         self.depotPath = settings['depot-paths'][0]
1096         if len(self.origin) == 0:
1097             self.origin = upstream
1098
1099         if self.preserveUser:
1100             if not self.canChangeChangelists():
1101                 die("Cannot preserve user names without p4 super-user or admin permissions")
1102
1103         if self.verbose:
1104             print "Origin branch is " + self.origin
1105
1106         if len(self.depotPath) == 0:
1107             print "Internal error: cannot locate perforce depot path from existing branches"
1108             sys.exit(128)
1109
1110         self.clientPath = p4Where(self.depotPath)
1111
1112         if len(self.clientPath) == 0:
1113             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1114             sys.exit(128)
1115
1116         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1117         self.oldWorkingDirectory = os.getcwd()
1118
1119         # ensure the clientPath exists
1120         if not os.path.exists(self.clientPath):
1121             os.makedirs(self.clientPath)
1122
1123         chdir(self.clientPath)
1124         print "Synchronizing p4 checkout..."
1125         p4_sync("...")
1126         self.check()
1127
1128         commits = []
1129         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1130             commits.append(line.strip())
1131         commits.reverse()
1132
1133         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1134             self.checkAuthorship = False
1135         else:
1136             self.checkAuthorship = True
1137
1138         if self.preserveUser:
1139             self.checkValidP4Users(commits)
1140
1141         while len(commits) > 0:
1142             commit = commits[0]
1143             commits = commits[1:]
1144             self.applyCommit(commit)
1145             if not self.interactive:
1146                 break
1147
1148         if len(commits) == 0:
1149             print "All changes applied!"
1150             chdir(self.oldWorkingDirectory)
1151
1152             sync = P4Sync()
1153             sync.run([])
1154
1155             rebase = P4Rebase()
1156             rebase.rebase()
1157
1158         return True
1159
1160 class P4Sync(Command, P4UserMap):
1161     delete_actions = ( "delete", "move/delete", "purge" )
1162
1163     def __init__(self):
1164         Command.__init__(self)
1165         P4UserMap.__init__(self)
1166         self.options = [
1167                 optparse.make_option("--branch", dest="branch"),
1168                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1169                 optparse.make_option("--changesfile", dest="changesFile"),
1170                 optparse.make_option("--silent", dest="silent", action="store_true"),
1171                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1172                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1173                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1174                                      help="Import into refs/heads/ , not refs/remotes"),
1175                 optparse.make_option("--max-changes", dest="maxChanges"),
1176                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1177                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1178                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1179                                      help="Only sync files that are included in the Perforce Client Spec")
1180         ]
1181         self.description = """Imports from Perforce into a git repository.\n
1182     example:
1183     //depot/my/project/ -- to import the current head
1184     //depot/my/project/@all -- to import everything
1185     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1186
1187     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1188
1189         self.usage += " //depot/path[@revRange]"
1190         self.silent = False
1191         self.createdBranches = set()
1192         self.committedChanges = set()
1193         self.branch = ""
1194         self.detectBranches = False
1195         self.detectLabels = False
1196         self.changesFile = ""
1197         self.syncWithOrigin = True
1198         self.verbose = False
1199         self.importIntoRemotes = True
1200         self.maxChanges = ""
1201         self.isWindows = (platform.system() == "Windows")
1202         self.keepRepoPath = False
1203         self.depotPaths = None
1204         self.p4BranchesInGit = []
1205         self.cloneExclude = []
1206         self.useClientSpec = False
1207         self.clientSpecDirs = []
1208
1209         if gitConfig("git-p4.syncFromOrigin") == "false":
1210             self.syncWithOrigin = False
1211
1212     #
1213     # P4 wildcards are not allowed in filenames.  P4 complains
1214     # if you simply add them, but you can force it with "-f", in
1215     # which case it translates them into %xx encoding internally.
1216     # Search for and fix just these four characters.  Do % last so
1217     # that fixing it does not inadvertently create new %-escapes.
1218     #
1219     def wildcard_decode(self, path):
1220         # Cannot have * in a filename in windows; untested as to
1221         # what p4 would do in such a case.
1222         if not self.isWindows:
1223             path = path.replace("%2A", "*")
1224         path = path.replace("%23", "#") \
1225                    .replace("%40", "@") \
1226                    .replace("%25", "%")
1227         return path
1228
1229     def extractFilesFromCommit(self, commit):
1230         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1231                              for path in self.cloneExclude]
1232         files = []
1233         fnum = 0
1234         while commit.has_key("depotFile%s" % fnum):
1235             path =  commit["depotFile%s" % fnum]
1236
1237             if [p for p in self.cloneExclude
1238                 if p4PathStartsWith(path, p)]:
1239                 found = False
1240             else:
1241                 found = [p for p in self.depotPaths
1242                          if p4PathStartsWith(path, p)]
1243             if not found:
1244                 fnum = fnum + 1
1245                 continue
1246
1247             file = {}
1248             file["path"] = path
1249             file["rev"] = commit["rev%s" % fnum]
1250             file["action"] = commit["action%s" % fnum]
1251             file["type"] = commit["type%s" % fnum]
1252             files.append(file)
1253             fnum = fnum + 1
1254         return files
1255
1256     def stripRepoPath(self, path, prefixes):
1257         if self.useClientSpec:
1258
1259             # if using the client spec, we use the output directory
1260             # specified in the client.  For example, a view
1261             #   //depot/foo/branch/... //client/branch/foo/...
1262             # will end up putting all foo/branch files into
1263             #  branch/foo/
1264             for val in self.clientSpecDirs:
1265                 if path.startswith(val[0]):
1266                     # replace the depot path with the client path
1267                     path = path.replace(val[0], val[1][1])
1268                     # now strip out the client (//client/...)
1269                     path = re.sub("^(//[^/]+/)", '', path)
1270                     # the rest is all path
1271                     return path
1272
1273         if self.keepRepoPath:
1274             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1275
1276         for p in prefixes:
1277             if p4PathStartsWith(path, p):
1278                 path = path[len(p):]
1279
1280         return path
1281
1282     def splitFilesIntoBranches(self, commit):
1283         branches = {}
1284         fnum = 0
1285         while commit.has_key("depotFile%s" % fnum):
1286             path =  commit["depotFile%s" % fnum]
1287             found = [p for p in self.depotPaths
1288                      if p4PathStartsWith(path, p)]
1289             if not found:
1290                 fnum = fnum + 1
1291                 continue
1292
1293             file = {}
1294             file["path"] = path
1295             file["rev"] = commit["rev%s" % fnum]
1296             file["action"] = commit["action%s" % fnum]
1297             file["type"] = commit["type%s" % fnum]
1298             fnum = fnum + 1
1299
1300             relPath = self.stripRepoPath(path, self.depotPaths)
1301
1302             for branch in self.knownBranches.keys():
1303
1304                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1305                 if relPath.startswith(branch + "/"):
1306                     if branch not in branches:
1307                         branches[branch] = []
1308                     branches[branch].append(file)
1309                     break
1310
1311         return branches
1312
1313     # output one file from the P4 stream
1314     # - helper for streamP4Files
1315
1316     def streamOneP4File(self, file, contents):
1317         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1318         relPath = self.wildcard_decode(relPath)
1319         if verbose:
1320             sys.stderr.write("%s\n" % relPath)
1321
1322         (type_base, type_mods) = split_p4_type(file["type"])
1323
1324         git_mode = "100644"
1325         if "x" in type_mods:
1326             git_mode = "100755"
1327         if type_base == "symlink":
1328             git_mode = "120000"
1329             # p4 print on a symlink contains "target\n"; remove the newline
1330             data = ''.join(contents)
1331             contents = [data[:-1]]
1332
1333         if type_base == "utf16":
1334             # p4 delivers different text in the python output to -G
1335             # than it does when using "print -o", or normal p4 client
1336             # operations.  utf16 is converted to ascii or utf8, perhaps.
1337             # But ascii text saved as -t utf16 is completely mangled.
1338             # Invoke print -o to get the real contents.
1339             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1340             contents = [ text ]
1341
1342         if type_base == "apple":
1343             # Apple filetype files will be streamed as a concatenation of
1344             # its appledouble header and the contents.  This is useless
1345             # on both macs and non-macs.  If using "print -q -o xx", it
1346             # will create "xx" with the data, and "%xx" with the header.
1347             # This is also not very useful.
1348             #
1349             # Ideally, someday, this script can learn how to generate
1350             # appledouble files directly and import those to git, but
1351             # non-mac machines can never find a use for apple filetype.
1352             print "\nIgnoring apple filetype file %s" % file['depotFile']
1353             return
1354
1355         # Perhaps windows wants unicode, utf16 newlines translated too;
1356         # but this is not doing it.
1357         if self.isWindows and type_base == "text":
1358             mangled = []
1359             for data in contents:
1360                 data = data.replace("\r\n", "\n")
1361                 mangled.append(data)
1362             contents = mangled
1363
1364         # Note that we do not try to de-mangle keywords on utf16 files,
1365         # even though in theory somebody may want that.
1366         if type_base in ("text", "unicode", "binary"):
1367             if "ko" in type_mods:
1368                 text = ''.join(contents)
1369                 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1370                 contents = [ text ]
1371             elif "k" in type_mods:
1372                 text = ''.join(contents)
1373                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1374                 contents = [ text ]
1375
1376         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1377
1378         # total length...
1379         length = 0
1380         for d in contents:
1381             length = length + len(d)
1382
1383         self.gitStream.write("data %d\n" % length)
1384         for d in contents:
1385             self.gitStream.write(d)
1386         self.gitStream.write("\n")
1387
1388     def streamOneP4Deletion(self, file):
1389         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1390         if verbose:
1391             sys.stderr.write("delete %s\n" % relPath)
1392         self.gitStream.write("D %s\n" % relPath)
1393
1394     # handle another chunk of streaming data
1395     def streamP4FilesCb(self, marshalled):
1396
1397         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1398             # start of a new file - output the old one first
1399             self.streamOneP4File(self.stream_file, self.stream_contents)
1400             self.stream_file = {}
1401             self.stream_contents = []
1402             self.stream_have_file_info = False
1403
1404         # pick up the new file information... for the
1405         # 'data' field we need to append to our array
1406         for k in marshalled.keys():
1407             if k == 'data':
1408                 self.stream_contents.append(marshalled['data'])
1409             else:
1410                 self.stream_file[k] = marshalled[k]
1411
1412         self.stream_have_file_info = True
1413
1414     # Stream directly from "p4 files" into "git fast-import"
1415     def streamP4Files(self, files):
1416         filesForCommit = []
1417         filesToRead = []
1418         filesToDelete = []
1419
1420         for f in files:
1421             includeFile = True
1422             for val in self.clientSpecDirs:
1423                 if f['path'].startswith(val[0]):
1424                     if val[1][0] <= 0:
1425                         includeFile = False
1426                     break
1427
1428             if includeFile:
1429                 filesForCommit.append(f)
1430                 if f['action'] in self.delete_actions:
1431                     filesToDelete.append(f)
1432                 else:
1433                     filesToRead.append(f)
1434
1435         # deleted files...
1436         for f in filesToDelete:
1437             self.streamOneP4Deletion(f)
1438
1439         if len(filesToRead) > 0:
1440             self.stream_file = {}
1441             self.stream_contents = []
1442             self.stream_have_file_info = False
1443
1444             # curry self argument
1445             def streamP4FilesCbSelf(entry):
1446                 self.streamP4FilesCb(entry)
1447
1448             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1449
1450             p4CmdList(["-x", "-", "print"],
1451                       stdin=fileArgs,
1452                       cb=streamP4FilesCbSelf)
1453
1454             # do the last chunk
1455             if self.stream_file.has_key('depotFile'):
1456                 self.streamOneP4File(self.stream_file, self.stream_contents)
1457
1458     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1459         epoch = details["time"]
1460         author = details["user"]
1461         self.branchPrefixes = branchPrefixes
1462
1463         if self.verbose:
1464             print "commit into %s" % branch
1465
1466         # start with reading files; if that fails, we should not
1467         # create a commit.
1468         new_files = []
1469         for f in files:
1470             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1471                 new_files.append (f)
1472             else:
1473                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1474
1475         self.gitStream.write("commit %s\n" % branch)
1476 #        gitStream.write("mark :%s\n" % details["change"])
1477         self.committedChanges.add(int(details["change"]))
1478         committer = ""
1479         if author not in self.users:
1480             self.getUserMapFromPerforceServer()
1481         if author in self.users:
1482             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1483         else:
1484             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1485
1486         self.gitStream.write("committer %s\n" % committer)
1487
1488         self.gitStream.write("data <<EOT\n")
1489         self.gitStream.write(details["desc"])
1490         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1491                              % (','.join (branchPrefixes), details["change"]))
1492         if len(details['options']) > 0:
1493             self.gitStream.write(": options = %s" % details['options'])
1494         self.gitStream.write("]\nEOT\n\n")
1495
1496         if len(parent) > 0:
1497             if self.verbose:
1498                 print "parent %s" % parent
1499             self.gitStream.write("from %s\n" % parent)
1500
1501         self.streamP4Files(new_files)
1502         self.gitStream.write("\n")
1503
1504         change = int(details["change"])
1505
1506         if self.labels.has_key(change):
1507             label = self.labels[change]
1508             labelDetails = label[0]
1509             labelRevisions = label[1]
1510             if self.verbose:
1511                 print "Change %s is labelled %s" % (change, labelDetails)
1512
1513             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1514                                                     for p in branchPrefixes])
1515
1516             if len(files) == len(labelRevisions):
1517
1518                 cleanedFiles = {}
1519                 for info in files:
1520                     if info["action"] in self.delete_actions:
1521                         continue
1522                     cleanedFiles[info["depotFile"]] = info["rev"]
1523
1524                 if cleanedFiles == labelRevisions:
1525                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1526                     self.gitStream.write("from %s\n" % branch)
1527
1528                     owner = labelDetails["Owner"]
1529                     tagger = ""
1530                     if author in self.users:
1531                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1532                     else:
1533                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1534                     self.gitStream.write("tagger %s\n" % tagger)
1535                     self.gitStream.write("data <<EOT\n")
1536                     self.gitStream.write(labelDetails["Description"])
1537                     self.gitStream.write("EOT\n\n")
1538
1539                 else:
1540                     if not self.silent:
1541                         print ("Tag %s does not match with change %s: files do not match."
1542                                % (labelDetails["label"], change))
1543
1544             else:
1545                 if not self.silent:
1546                     print ("Tag %s does not match with change %s: file count is different."
1547                            % (labelDetails["label"], change))
1548
1549     def getLabels(self):
1550         self.labels = {}
1551
1552         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1553         if len(l) > 0 and not self.silent:
1554             print "Finding files belonging to labels in %s" % `self.depotPaths`
1555
1556         for output in l:
1557             label = output["label"]
1558             revisions = {}
1559             newestChange = 0
1560             if self.verbose:
1561                 print "Querying files for label %s" % label
1562             for file in p4CmdList(["files"] +
1563                                       ["%s...@%s" % (p, label)
1564                                           for p in self.depotPaths]):
1565                 revisions[file["depotFile"]] = file["rev"]
1566                 change = int(file["change"])
1567                 if change > newestChange:
1568                     newestChange = change
1569
1570             self.labels[newestChange] = [output, revisions]
1571
1572         if self.verbose:
1573             print "Label changes: %s" % self.labels.keys()
1574
1575     def guessProjectName(self):
1576         for p in self.depotPaths:
1577             if p.endswith("/"):
1578                 p = p[:-1]
1579             p = p[p.strip().rfind("/") + 1:]
1580             if not p.endswith("/"):
1581                p += "/"
1582             return p
1583
1584     def getBranchMapping(self):
1585         lostAndFoundBranches = set()
1586
1587         user = gitConfig("git-p4.branchUser")
1588         if len(user) > 0:
1589             command = "branches -u %s" % user
1590         else:
1591             command = "branches"
1592
1593         for info in p4CmdList(command):
1594             details = p4Cmd("branch -o %s" % info["branch"])
1595             viewIdx = 0
1596             while details.has_key("View%s" % viewIdx):
1597                 paths = details["View%s" % viewIdx].split(" ")
1598                 viewIdx = viewIdx + 1
1599                 # require standard //depot/foo/... //depot/bar/... mapping
1600                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1601                     continue
1602                 source = paths[0]
1603                 destination = paths[1]
1604                 ## HACK
1605                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1606                     source = source[len(self.depotPaths[0]):-4]
1607                     destination = destination[len(self.depotPaths[0]):-4]
1608
1609                     if destination in self.knownBranches:
1610                         if not self.silent:
1611                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1612                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1613                         continue
1614
1615                     self.knownBranches[destination] = source
1616
1617                     lostAndFoundBranches.discard(destination)
1618
1619                     if source not in self.knownBranches:
1620                         lostAndFoundBranches.add(source)
1621
1622         # Perforce does not strictly require branches to be defined, so we also
1623         # check git config for a branch list.
1624         #
1625         # Example of branch definition in git config file:
1626         # [git-p4]
1627         #   branchList=main:branchA
1628         #   branchList=main:branchB
1629         #   branchList=branchA:branchC
1630         configBranches = gitConfigList("git-p4.branchList")
1631         for branch in configBranches:
1632             if branch:
1633                 (source, destination) = branch.split(":")
1634                 self.knownBranches[destination] = source
1635
1636                 lostAndFoundBranches.discard(destination)
1637
1638                 if source not in self.knownBranches:
1639                     lostAndFoundBranches.add(source)
1640
1641
1642         for branch in lostAndFoundBranches:
1643             self.knownBranches[branch] = branch
1644
1645     def getBranchMappingFromGitBranches(self):
1646         branches = p4BranchesInGit(self.importIntoRemotes)
1647         for branch in branches.keys():
1648             if branch == "master":
1649                 branch = "main"
1650             else:
1651                 branch = branch[len(self.projectName):]
1652             self.knownBranches[branch] = branch
1653
1654     def listExistingP4GitBranches(self):
1655         # branches holds mapping from name to commit
1656         branches = p4BranchesInGit(self.importIntoRemotes)
1657         self.p4BranchesInGit = branches.keys()
1658         for branch in branches.keys():
1659             self.initialParents[self.refPrefix + branch] = branches[branch]
1660
1661     def updateOptionDict(self, d):
1662         option_keys = {}
1663         if self.keepRepoPath:
1664             option_keys['keepRepoPath'] = 1
1665
1666         d["options"] = ' '.join(sorted(option_keys.keys()))
1667
1668     def readOptions(self, d):
1669         self.keepRepoPath = (d.has_key('options')
1670                              and ('keepRepoPath' in d['options']))
1671
1672     def gitRefForBranch(self, branch):
1673         if branch == "main":
1674             return self.refPrefix + "master"
1675
1676         if len(branch) <= 0:
1677             return branch
1678
1679         return self.refPrefix + self.projectName + branch
1680
1681     def gitCommitByP4Change(self, ref, change):
1682         if self.verbose:
1683             print "looking in ref " + ref + " for change %s using bisect..." % change
1684
1685         earliestCommit = ""
1686         latestCommit = parseRevision(ref)
1687
1688         while True:
1689             if self.verbose:
1690                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1691             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1692             if len(next) == 0:
1693                 if self.verbose:
1694                     print "argh"
1695                 return ""
1696             log = extractLogMessageFromGitCommit(next)
1697             settings = extractSettingsGitLog(log)
1698             currentChange = int(settings['change'])
1699             if self.verbose:
1700                 print "current change %s" % currentChange
1701
1702             if currentChange == change:
1703                 if self.verbose:
1704                     print "found %s" % next
1705                 return next
1706
1707             if currentChange < change:
1708                 earliestCommit = "^%s" % next
1709             else:
1710                 latestCommit = "%s" % next
1711
1712         return ""
1713
1714     def importNewBranch(self, branch, maxChange):
1715         # make fast-import flush all changes to disk and update the refs using the checkpoint
1716         # command so that we can try to find the branch parent in the git history
1717         self.gitStream.write("checkpoint\n\n");
1718         self.gitStream.flush();
1719         branchPrefix = self.depotPaths[0] + branch + "/"
1720         range = "@1,%s" % maxChange
1721         #print "prefix" + branchPrefix
1722         changes = p4ChangesForPaths([branchPrefix], range)
1723         if len(changes) <= 0:
1724             return False
1725         firstChange = changes[0]
1726         #print "first change in branch: %s" % firstChange
1727         sourceBranch = self.knownBranches[branch]
1728         sourceDepotPath = self.depotPaths[0] + sourceBranch
1729         sourceRef = self.gitRefForBranch(sourceBranch)
1730         #print "source " + sourceBranch
1731
1732         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1733         #print "branch parent: %s" % branchParentChange
1734         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1735         if len(gitParent) > 0:
1736             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1737             #print "parent git commit: %s" % gitParent
1738
1739         self.importChanges(changes)
1740         return True
1741
1742     def importChanges(self, changes):
1743         cnt = 1
1744         for change in changes:
1745             description = p4Cmd("describe %s" % change)
1746             self.updateOptionDict(description)
1747
1748             if not self.silent:
1749                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1750                 sys.stdout.flush()
1751             cnt = cnt + 1
1752
1753             try:
1754                 if self.detectBranches:
1755                     branches = self.splitFilesIntoBranches(description)
1756                     for branch in branches.keys():
1757                         ## HACK  --hwn
1758                         branchPrefix = self.depotPaths[0] + branch + "/"
1759
1760                         parent = ""
1761
1762                         filesForCommit = branches[branch]
1763
1764                         if self.verbose:
1765                             print "branch is %s" % branch
1766
1767                         self.updatedBranches.add(branch)
1768
1769                         if branch not in self.createdBranches:
1770                             self.createdBranches.add(branch)
1771                             parent = self.knownBranches[branch]
1772                             if parent == branch:
1773                                 parent = ""
1774                             else:
1775                                 fullBranch = self.projectName + branch
1776                                 if fullBranch not in self.p4BranchesInGit:
1777                                     if not self.silent:
1778                                         print("\n    Importing new branch %s" % fullBranch);
1779                                     if self.importNewBranch(branch, change - 1):
1780                                         parent = ""
1781                                         self.p4BranchesInGit.append(fullBranch)
1782                                     if not self.silent:
1783                                         print("\n    Resuming with change %s" % change);
1784
1785                                 if self.verbose:
1786                                     print "parent determined through known branches: %s" % parent
1787
1788                         branch = self.gitRefForBranch(branch)
1789                         parent = self.gitRefForBranch(parent)
1790
1791                         if self.verbose:
1792                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1793
1794                         if len(parent) == 0 and branch in self.initialParents:
1795                             parent = self.initialParents[branch]
1796                             del self.initialParents[branch]
1797
1798                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1799                 else:
1800                     files = self.extractFilesFromCommit(description)
1801                     self.commit(description, files, self.branch, self.depotPaths,
1802                                 self.initialParent)
1803                     self.initialParent = ""
1804             except IOError:
1805                 print self.gitError.read()
1806                 sys.exit(1)
1807
1808     def importHeadRevision(self, revision):
1809         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1810
1811         details = {}
1812         details["user"] = "git perforce import user"
1813         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1814                            % (' '.join(self.depotPaths), revision))
1815         details["change"] = revision
1816         newestRevision = 0
1817
1818         fileCnt = 0
1819         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
1820
1821         for info in p4CmdList(["files"] + fileArgs):
1822
1823             if 'code' in info and info['code'] == 'error':
1824                 sys.stderr.write("p4 returned an error: %s\n"
1825                                  % info['data'])
1826                 if info['data'].find("must refer to client") >= 0:
1827                     sys.stderr.write("This particular p4 error is misleading.\n")
1828                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1829                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1830                 sys.exit(1)
1831             if 'p4ExitCode' in info:
1832                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1833                 sys.exit(1)
1834
1835
1836             change = int(info["change"])
1837             if change > newestRevision:
1838                 newestRevision = change
1839
1840             if info["action"] in self.delete_actions:
1841                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1842                 #fileCnt = fileCnt + 1
1843                 continue
1844
1845             for prop in ["depotFile", "rev", "action", "type" ]:
1846                 details["%s%s" % (prop, fileCnt)] = info[prop]
1847
1848             fileCnt = fileCnt + 1
1849
1850         details["change"] = newestRevision
1851
1852         # Use time from top-most change so that all git-p4 clones of
1853         # the same p4 repo have the same commit SHA1s.
1854         res = p4CmdList("describe -s %d" % newestRevision)
1855         newestTime = None
1856         for r in res:
1857             if r.has_key('time'):
1858                 newestTime = int(r['time'])
1859         if newestTime is None:
1860             die("\"describe -s\" on newest change %d did not give a time")
1861         details["time"] = newestTime
1862
1863         self.updateOptionDict(details)
1864         try:
1865             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1866         except IOError:
1867             print "IO error with git fast-import. Is your git version recent enough?"
1868             print self.gitError.read()
1869
1870
1871     def getClientSpec(self):
1872         specList = p4CmdList( "client -o" )
1873         temp = {}
1874         for entry in specList:
1875             for k,v in entry.iteritems():
1876                 if k.startswith("View"):
1877
1878                     # p4 has these %%1 to %%9 arguments in specs to
1879                     # reorder paths; which we can't handle (yet :)
1880                     if re.match('%%\d', v) != None:
1881                         print "Sorry, can't handle %%n arguments in client specs"
1882                         sys.exit(1)
1883
1884                     if v.startswith('"'):
1885                         start = 1
1886                     else:
1887                         start = 0
1888                     index = v.find("...")
1889
1890                     # save the "client view"; i.e the RHS of the view
1891                     # line that tells the client where to put the
1892                     # files for this view.
1893                     cv = v[index+3:].strip() # +3 to remove previous '...'
1894
1895                     # if the client view doesn't end with a
1896                     # ... wildcard, then we're going to mess up the
1897                     # output directory, so fail gracefully.
1898                     if not cv.endswith('...'):
1899                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1900                         sys.exit(1)
1901                     cv=cv[:-3]
1902
1903                     # now save the view; +index means included, -index
1904                     # means it should be filtered out.
1905                     v = v[start:index]
1906                     if v.startswith("-"):
1907                         v = v[1:]
1908                         include = -len(v)
1909                     else:
1910                         include = len(v)
1911
1912                     temp[v] = (include, cv)
1913
1914         self.clientSpecDirs = temp.items()
1915         self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1916
1917     def run(self, args):
1918         self.depotPaths = []
1919         self.changeRange = ""
1920         self.initialParent = ""
1921         self.previousDepotPaths = []
1922
1923         # map from branch depot path to parent branch
1924         self.knownBranches = {}
1925         self.initialParents = {}
1926         self.hasOrigin = originP4BranchesExist()
1927         if not self.syncWithOrigin:
1928             self.hasOrigin = False
1929
1930         if self.importIntoRemotes:
1931             self.refPrefix = "refs/remotes/p4/"
1932         else:
1933             self.refPrefix = "refs/heads/p4/"
1934
1935         if self.syncWithOrigin and self.hasOrigin:
1936             if not self.silent:
1937                 print "Syncing with origin first by calling git fetch origin"
1938             system("git fetch origin")
1939
1940         if len(self.branch) == 0:
1941             self.branch = self.refPrefix + "master"
1942             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1943                 system("git update-ref %s refs/heads/p4" % self.branch)
1944                 system("git branch -D p4");
1945             # create it /after/ importing, when master exists
1946             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1947                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1948
1949         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1950             self.getClientSpec()
1951
1952         # TODO: should always look at previous commits,
1953         # merge with previous imports, if possible.
1954         if args == []:
1955             if self.hasOrigin:
1956                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1957             self.listExistingP4GitBranches()
1958
1959             if len(self.p4BranchesInGit) > 1:
1960                 if not self.silent:
1961                     print "Importing from/into multiple branches"
1962                 self.detectBranches = True
1963
1964             if self.verbose:
1965                 print "branches: %s" % self.p4BranchesInGit
1966
1967             p4Change = 0
1968             for branch in self.p4BranchesInGit:
1969                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1970
1971                 settings = extractSettingsGitLog(logMsg)
1972
1973                 self.readOptions(settings)
1974                 if (settings.has_key('depot-paths')
1975                     and settings.has_key ('change')):
1976                     change = int(settings['change']) + 1
1977                     p4Change = max(p4Change, change)
1978
1979                     depotPaths = sorted(settings['depot-paths'])
1980                     if self.previousDepotPaths == []:
1981                         self.previousDepotPaths = depotPaths
1982                     else:
1983                         paths = []
1984                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1985                             prev_list = prev.split("/")
1986                             cur_list = cur.split("/")
1987                             for i in range(0, min(len(cur_list), len(prev_list))):
1988                                 if cur_list[i] <> prev_list[i]:
1989                                     i = i - 1
1990                                     break
1991
1992                             paths.append ("/".join(cur_list[:i + 1]))
1993
1994                         self.previousDepotPaths = paths
1995
1996             if p4Change > 0:
1997                 self.depotPaths = sorted(self.previousDepotPaths)
1998                 self.changeRange = "@%s,#head" % p4Change
1999                 if not self.detectBranches:
2000                     self.initialParent = parseRevision(self.branch)
2001                 if not self.silent and not self.detectBranches:
2002                     print "Performing incremental import into %s git branch" % self.branch
2003
2004         if not self.branch.startswith("refs/"):
2005             self.branch = "refs/heads/" + self.branch
2006
2007         if len(args) == 0 and self.depotPaths:
2008             if not self.silent:
2009                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2010         else:
2011             if self.depotPaths and self.depotPaths != args:
2012                 print ("previous import used depot path %s and now %s was specified. "
2013                        "This doesn't work!" % (' '.join (self.depotPaths),
2014                                                ' '.join (args)))
2015                 sys.exit(1)
2016
2017             self.depotPaths = sorted(args)
2018
2019         revision = ""
2020         self.users = {}
2021
2022         newPaths = []
2023         for p in self.depotPaths:
2024             if p.find("@") != -1:
2025                 atIdx = p.index("@")
2026                 self.changeRange = p[atIdx:]
2027                 if self.changeRange == "@all":
2028                     self.changeRange = ""
2029                 elif ',' not in self.changeRange:
2030                     revision = self.changeRange
2031                     self.changeRange = ""
2032                 p = p[:atIdx]
2033             elif p.find("#") != -1:
2034                 hashIdx = p.index("#")
2035                 revision = p[hashIdx:]
2036                 p = p[:hashIdx]
2037             elif self.previousDepotPaths == []:
2038                 revision = "#head"
2039
2040             p = re.sub ("\.\.\.$", "", p)
2041             if not p.endswith("/"):
2042                 p += "/"
2043
2044             newPaths.append(p)
2045
2046         self.depotPaths = newPaths
2047
2048
2049         self.loadUserMapFromCache()
2050         self.labels = {}
2051         if self.detectLabels:
2052             self.getLabels();
2053
2054         if self.detectBranches:
2055             ## FIXME - what's a P4 projectName ?
2056             self.projectName = self.guessProjectName()
2057
2058             if self.hasOrigin:
2059                 self.getBranchMappingFromGitBranches()
2060             else:
2061                 self.getBranchMapping()
2062             if self.verbose:
2063                 print "p4-git branches: %s" % self.p4BranchesInGit
2064                 print "initial parents: %s" % self.initialParents
2065             for b in self.p4BranchesInGit:
2066                 if b != "master":
2067
2068                     ## FIXME
2069                     b = b[len(self.projectName):]
2070                 self.createdBranches.add(b)
2071
2072         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2073
2074         importProcess = subprocess.Popen(["git", "fast-import"],
2075                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2076                                          stderr=subprocess.PIPE);
2077         self.gitOutput = importProcess.stdout
2078         self.gitStream = importProcess.stdin
2079         self.gitError = importProcess.stderr
2080
2081         if revision:
2082             self.importHeadRevision(revision)
2083         else:
2084             changes = []
2085
2086             if len(self.changesFile) > 0:
2087                 output = open(self.changesFile).readlines()
2088                 changeSet = set()
2089                 for line in output:
2090                     changeSet.add(int(line))
2091
2092                 for change in changeSet:
2093                     changes.append(change)
2094
2095                 changes.sort()
2096             else:
2097                 # catch "git-p4 sync" with no new branches, in a repo that
2098                 # does not have any existing git-p4 branches
2099                 if len(args) == 0 and not self.p4BranchesInGit:
2100                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2101                 if self.verbose:
2102                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2103                                                               self.changeRange)
2104                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2105
2106                 if len(self.maxChanges) > 0:
2107                     changes = changes[:min(int(self.maxChanges), len(changes))]
2108
2109             if len(changes) == 0:
2110                 if not self.silent:
2111                     print "No changes to import!"
2112                 return True
2113
2114             if not self.silent and not self.detectBranches:
2115                 print "Import destination: %s" % self.branch
2116
2117             self.updatedBranches = set()
2118
2119             self.importChanges(changes)
2120
2121             if not self.silent:
2122                 print ""
2123                 if len(self.updatedBranches) > 0:
2124                     sys.stdout.write("Updated branches: ")
2125                     for b in self.updatedBranches:
2126                         sys.stdout.write("%s " % b)
2127                     sys.stdout.write("\n")
2128
2129         self.gitStream.close()
2130         if importProcess.wait() != 0:
2131             die("fast-import failed: %s" % self.gitError.read())
2132         self.gitOutput.close()
2133         self.gitError.close()
2134
2135         return True
2136
2137 class P4Rebase(Command):
2138     def __init__(self):
2139         Command.__init__(self)
2140         self.options = [ ]
2141         self.description = ("Fetches the latest revision from perforce and "
2142                             + "rebases the current work (branch) against it")
2143         self.verbose = False
2144
2145     def run(self, args):
2146         sync = P4Sync()
2147         sync.run([])
2148
2149         return self.rebase()
2150
2151     def rebase(self):
2152         if os.system("git update-index --refresh") != 0:
2153             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2154         if len(read_pipe("git diff-index HEAD --")) > 0:
2155             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2156
2157         [upstream, settings] = findUpstreamBranchPoint()
2158         if len(upstream) == 0:
2159             die("Cannot find upstream branchpoint for rebase")
2160
2161         # the branchpoint may be p4/foo~3, so strip off the parent
2162         upstream = re.sub("~[0-9]+$", "", upstream)
2163
2164         print "Rebasing the current branch onto %s" % upstream
2165         oldHead = read_pipe("git rev-parse HEAD").strip()
2166         system("git rebase %s" % upstream)
2167         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2168         return True
2169
2170 class P4Clone(P4Sync):
2171     def __init__(self):
2172         P4Sync.__init__(self)
2173         self.description = "Creates a new git repository and imports from Perforce into it"
2174         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2175         self.options += [
2176             optparse.make_option("--destination", dest="cloneDestination",
2177                                  action='store', default=None,
2178                                  help="where to leave result of the clone"),
2179             optparse.make_option("-/", dest="cloneExclude",
2180                                  action="append", type="string",
2181                                  help="exclude depot path"),
2182             optparse.make_option("--bare", dest="cloneBare",
2183                                  action="store_true", default=False),
2184         ]
2185         self.cloneDestination = None
2186         self.needsGit = False
2187         self.cloneBare = False
2188
2189     # This is required for the "append" cloneExclude action
2190     def ensure_value(self, attr, value):
2191         if not hasattr(self, attr) or getattr(self, attr) is None:
2192             setattr(self, attr, value)
2193         return getattr(self, attr)
2194
2195     def defaultDestination(self, args):
2196         ## TODO: use common prefix of args?
2197         depotPath = args[0]
2198         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2199         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2200         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2201         depotDir = re.sub(r"/$", "", depotDir)
2202         return os.path.split(depotDir)[1]
2203
2204     def run(self, args):
2205         if len(args) < 1:
2206             return False
2207
2208         if self.keepRepoPath and not self.cloneDestination:
2209             sys.stderr.write("Must specify destination for --keep-path\n")
2210             sys.exit(1)
2211
2212         depotPaths = args
2213
2214         if not self.cloneDestination and len(depotPaths) > 1:
2215             self.cloneDestination = depotPaths[-1]
2216             depotPaths = depotPaths[:-1]
2217
2218         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2219         for p in depotPaths:
2220             if not p.startswith("//"):
2221                 return False
2222
2223         if not self.cloneDestination:
2224             self.cloneDestination = self.defaultDestination(args)
2225
2226         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2227
2228         if not os.path.exists(self.cloneDestination):
2229             os.makedirs(self.cloneDestination)
2230         chdir(self.cloneDestination)
2231
2232         init_cmd = [ "git", "init" ]
2233         if self.cloneBare:
2234             init_cmd.append("--bare")
2235         subprocess.check_call(init_cmd)
2236
2237         if not P4Sync.run(self, depotPaths):
2238             return False
2239         if self.branch != "master":
2240             if self.importIntoRemotes:
2241                 masterbranch = "refs/remotes/p4/master"
2242             else:
2243                 masterbranch = "refs/heads/p4/master"
2244             if gitBranchExists(masterbranch):
2245                 system("git branch master %s" % masterbranch)
2246                 if not self.cloneBare:
2247                     system("git checkout -f")
2248             else:
2249                 print "Could not detect main branch. No checkout/master branch created."
2250
2251         return True
2252
2253 class P4Branches(Command):
2254     def __init__(self):
2255         Command.__init__(self)
2256         self.options = [ ]
2257         self.description = ("Shows the git branches that hold imports and their "
2258                             + "corresponding perforce depot paths")
2259         self.verbose = False
2260
2261     def run(self, args):
2262         if originP4BranchesExist():
2263             createOrUpdateBranchesFromOrigin()
2264
2265         cmdline = "git rev-parse --symbolic "
2266         cmdline += " --remotes"
2267
2268         for line in read_pipe_lines(cmdline):
2269             line = line.strip()
2270
2271             if not line.startswith('p4/') or line == "p4/HEAD":
2272                 continue
2273             branch = line
2274
2275             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2276             settings = extractSettingsGitLog(log)
2277
2278             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2279         return True
2280
2281 class HelpFormatter(optparse.IndentedHelpFormatter):
2282     def __init__(self):
2283         optparse.IndentedHelpFormatter.__init__(self)
2284
2285     def format_description(self, description):
2286         if description:
2287             return description + "\n"
2288         else:
2289             return ""
2290
2291 def printUsage(commands):
2292     print "usage: %s <command> [options]" % sys.argv[0]
2293     print ""
2294     print "valid commands: %s" % ", ".join(commands)
2295     print ""
2296     print "Try %s <command> --help for command specific help." % sys.argv[0]
2297     print ""
2298
2299 commands = {
2300     "debug" : P4Debug,
2301     "submit" : P4Submit,
2302     "commit" : P4Submit,
2303     "sync" : P4Sync,
2304     "rebase" : P4Rebase,
2305     "clone" : P4Clone,
2306     "rollback" : P4RollBack,
2307     "branches" : P4Branches
2308 }
2309
2310
2311 def main():
2312     if len(sys.argv[1:]) == 0:
2313         printUsage(commands.keys())
2314         sys.exit(2)
2315
2316     cmd = ""
2317     cmdName = sys.argv[1]
2318     try:
2319         klass = commands[cmdName]
2320         cmd = klass()
2321     except KeyError:
2322         print "unknown command %s" % cmdName
2323         print ""
2324         printUsage(commands.keys())
2325         sys.exit(2)
2326
2327     options = cmd.options
2328     cmd.gitdir = os.environ.get("GIT_DIR", None)
2329
2330     args = sys.argv[2:]
2331
2332     if len(options) > 0:
2333         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2334
2335         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2336                                        options,
2337                                        description = cmd.description,
2338                                        formatter = HelpFormatter())
2339
2340         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2341     global verbose
2342     verbose = cmd.verbose
2343     if cmd.needsGit:
2344         if cmd.gitdir == None:
2345             cmd.gitdir = os.path.abspath(".git")
2346             if not isValidGitDir(cmd.gitdir):
2347                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2348                 if os.path.exists(cmd.gitdir):
2349                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2350                     if len(cdup) > 0:
2351                         chdir(cdup);
2352
2353         if not isValidGitDir(cmd.gitdir):
2354             if isValidGitDir(cmd.gitdir + "/.git"):
2355                 cmd.gitdir += "/.git"
2356             else:
2357                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2358
2359         os.environ["GIT_DIR"] = cmd.gitdir
2360
2361     if not cmd.run(args):
2362         parser.print_help()
2363
2364
2365 if __name__ == '__main__':
2366     main()