contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = ["p4"]
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += ["-u",user]
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += ["-P", password]
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += ["-p", port]
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += ["-h", host]
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += ["-c", client]
  46
  47
  48     if isinstance(cmd,basestring):
  49         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50     else:
  51         real_cmd += cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56     # not using the shell, we have to set it ourselves.  This path could
  57     # be relative, so go there first, then figure out where we ended up.
  58     os.chdir(dir)
  59     os.environ['PWD'] = os.getcwd()
  60
  61 def die(msg):
  62     if verbose:
  63         raise Exception(msg)
  64     else:
  65         sys.stderr.write(msg + "\n")
  66         sys.exit(1)
  67
  68 def write_pipe(c, stdin):
  69     if verbose:
  70         sys.stderr.write('Writing pipe: %s\n' % str(c))
  71
  72     expand = isinstance(c,basestring)
  73     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  74     pipe = p.stdin
  75     val = pipe.write(stdin)
  76     pipe.close()
  77     if p.wait():
  78         die('Command failed: %s' % str(c))
  79
  80     return val
  81
  82 def p4_write_pipe(c, stdin):
  83     real_cmd = p4_build_cmd(c)
  84     return write_pipe(real_cmd, stdin)
  85
  86 def read_pipe(c, ignore_error=False):
  87     if verbose:
  88         sys.stderr.write('Reading pipe: %s\n' % str(c))
  89
  90     expand = isinstance(c,basestring)
  91     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  92     pipe = p.stdout
  93     val = pipe.read()
  94     if p.wait() and not ignore_error:
  95         die('Command failed: %s' % str(c))
  96
  97     return val
  98
  99 def p4_read_pipe(c, ignore_error=False):
 100     real_cmd = p4_build_cmd(c)
 101     return read_pipe(real_cmd, ignore_error)
 102
 103 def read_pipe_lines(c):
 104     if verbose:
 105         sys.stderr.write('Reading pipe: %s\n' % str(c))
 106
 107     expand = isinstance(c, basestring)
 108     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 109     pipe = p.stdout
 110     val = pipe.readlines()
 111     if pipe.close() or p.wait():
 112         die('Command failed: %s' % str(c))
 113
 114     return val
 115
 116 def p4_read_pipe_lines(c):
 117     """Specifically invoke p4 on the command supplied. """
 118     real_cmd = p4_build_cmd(c)
 119     return read_pipe_lines(real_cmd)
 120
 121 def system(cmd):
 122     expand = isinstance(cmd,basestring)
 123     if verbose:
 124         sys.stderr.write("executing %s\n" % str(cmd))
 125     subprocess.check_call(cmd, shell=expand)
 126
 127 def p4_system(cmd):
 128     """Specifically invoke p4 as the system command. """
 129     real_cmd = p4_build_cmd(cmd)
 130     expand = isinstance(real_cmd, basestring)
 131     subprocess.check_call(real_cmd, shell=expand)
 132
 133 def p4_integrate(src, dest):
 134     p4_system(["integrate", "-Dt", src, dest])
 135
 136 def p4_sync(path):
 137     p4_system(["sync", path])
 138
 139 def p4_add(f):
 140     p4_system(["add", f])
 141
 142 def p4_delete(f):
 143     p4_system(["delete", f])
 144
 145 def p4_edit(f):
 146     p4_system(["edit", f])
 147
 148 def p4_revert(f):
 149     p4_system(["revert", f])
 150
 151 def p4_reopen(type, file):
 152     p4_system(["reopen", "-t", type, file])
 153
 154 #
 155 # Canonicalize the p4 type and return a tuple of the
 156 # base type, plus any modifiers.  See "p4 help filetypes"
 157 # for a list and explanation.
 158 #
 159 def split_p4_type(p4type):
 160
 161     p4_filetypes_historical = {
 162         "ctempobj": "binary+Sw",
 163         "ctext": "text+C",
 164         "cxtext": "text+Cx",
 165         "ktext": "text+k",
 166         "kxtext": "text+kx",
 167         "ltext": "text+F",
 168         "tempobj": "binary+FSw",
 169         "ubinary": "binary+F",
 170         "uresource": "resource+F",
 171         "uxbinary": "binary+Fx",
 172         "xbinary": "binary+x",
 173         "xltext": "text+Fx",
 174         "xtempobj": "binary+Swx",
 175         "xtext": "text+x",
 176         "xunicode": "unicode+x",
 177         "xutf16": "utf16+x",
 178     }
 179     if p4type in p4_filetypes_historical:
 180         p4type = p4_filetypes_historical[p4type]
 181     mods = ""
 182     s = p4type.split("+")
 183     base = s[0]
 184     mods = ""
 185     if len(s) > 1:
 186         mods = s[1]
 187     return (base, mods)
 188
 189
 190 def setP4ExecBit(file, mode):
 191     # Reopens an already open file and changes the execute bit to match
 192     # the execute bit setting in the passed in mode.
 193
 194     p4Type = "+x"
 195
 196     if not isModeExec(mode):
 197         p4Type = getP4OpenedType(file)
 198         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 199         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 200         if p4Type[-1] == "+":
 201             p4Type = p4Type[0:-1]
 202
 203     p4_reopen(p4Type, file)
 204
 205 def getP4OpenedType(file):
 206     # Returns the perforce file type for the given file.
 207
 208     result = p4_read_pipe(["opened", file])
 209     match = re.match(".*\((.+)\)\r?$", result)
 210     if match:
 211         return match.group(1)
 212     else:
 213         die("Could not determine file type for %s (result: '%s')" % (file, result))
 214
 215 def diffTreePattern():
 216     # This is a simple generator for the diff tree regex pattern. This could be
 217     # a class variable if this and parseDiffTreeEntry were a part of a class.
 218     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 219     while True:
 220         yield pattern
 221
 222 def parseDiffTreeEntry(entry):
 223     """Parses a single diff tree entry into its component elements.
 224
 225     See git-diff-tree(1) manpage for details about the format of the diff
 226     output. This method returns a dictionary with the following elements:
 227
 228     src_mode - The mode of the source file
 229     dst_mode - The mode of the destination file
 230     src_sha1 - The sha1 for the source file
 231     dst_sha1 - The sha1 fr the destination file
 232     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 233     status_score - The score for the status (applicable for 'C' and 'R'
 234                    statuses). This is None if there is no score.
 235     src - The path for the source file.
 236     dst - The path for the destination file. This is only present for
 237           copy or renames. If it is not present, this is None.
 238
 239     If the pattern is not matched, None is returned."""
 240
 241     match = diffTreePattern().next().match(entry)
 242     if match:
 243         return {
 244             'src_mode': match.group(1),
 245             'dst_mode': match.group(2),
 246             'src_sha1': match.group(3),
 247             'dst_sha1': match.group(4),
 248             'status': match.group(5),
 249             'status_score': match.group(6),
 250             'src': match.group(7),
 251             'dst': match.group(10)
 252         }
 253     return None
 254
 255 def isModeExec(mode):
 256     # Returns True if the given git mode represents an executable file,
 257     # otherwise False.
 258     return mode[-3:] == "755"
 259
 260 def isModeExecChanged(src_mode, dst_mode):
 261     return isModeExec(src_mode) != isModeExec(dst_mode)
 262
 263 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 264
 265     if isinstance(cmd,basestring):
 266         cmd = "-G " + cmd
 267         expand = True
 268     else:
 269         cmd = ["-G"] + cmd
 270         expand = False
 271
 272     cmd = p4_build_cmd(cmd)
 273     if verbose:
 274         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 275
 276     # Use a temporary file to avoid deadlocks without
 277     # subprocess.communicate(), which would put another copy
 278     # of stdout into memory.
 279     stdin_file = None
 280     if stdin is not None:
 281         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 282         if isinstance(stdin,basestring):
 283             stdin_file.write(stdin)
 284         else:
 285             for i in stdin:
 286                 stdin_file.write(i + '\n')
 287         stdin_file.flush()
 288         stdin_file.seek(0)
 289
 290     p4 = subprocess.Popen(cmd,
 291                           shell=expand,
 292                           stdin=stdin_file,
 293                           stdout=subprocess.PIPE)
 294
 295     result = []
 296     try:
 297         while True:
 298             entry = marshal.load(p4.stdout)
 299             if cb is not None:
 300                 cb(entry)
 301             else:
 302                 result.append(entry)
 303     except EOFError:
 304         pass
 305     exitCode = p4.wait()
 306     if exitCode != 0:
 307         entry = {}
 308         entry["p4ExitCode"] = exitCode
 309         result.append(entry)
 310
 311     return result
 312
 313 def p4Cmd(cmd):
 314     list = p4CmdList(cmd)
 315     result = {}
 316     for entry in list:
 317         result.update(entry)
 318     return result;
 319
 320 def p4Where(depotPath):
 321     if not depotPath.endswith("/"):
 322         depotPath += "/"
 323     depotPath = depotPath + "..."
 324     outputList = p4CmdList(["where", depotPath])
 325     output = None
 326     for entry in outputList:
 327         if "depotFile" in entry:
 328             if entry["depotFile"] == depotPath:
 329                 output = entry
 330                 break
 331         elif "data" in entry:
 332             data = entry.get("data")
 333             space = data.find(" ")
 334             if data[:space] == depotPath:
 335                 output = entry
 336                 break
 337     if output == None:
 338         return ""
 339     if output["code"] == "error":
 340         return ""
 341     clientPath = ""
 342     if "path" in output:
 343         clientPath = output.get("path")
 344     elif "data" in output:
 345         data = output.get("data")
 346         lastSpace = data.rfind(" ")
 347         clientPath = data[lastSpace + 1:]
 348
 349     if clientPath.endswith("..."):
 350         clientPath = clientPath[:-3]
 351     return clientPath
 352
 353 def currentGitBranch():
 354     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 355
 356 def isValidGitDir(path):
 357     if (os.path.exists(path + "/HEAD")
 358         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 359         return True;
 360     return False
 361
 362 def parseRevision(ref):
 363     return read_pipe("git rev-parse %s" % ref).strip()
 364
 365 def extractLogMessageFromGitCommit(commit):
 366     logMessage = ""
 367
 368     ## fixme: title is first line of commit, not 1st paragraph.
 369     foundTitle = False
 370     for log in read_pipe_lines("git cat-file commit %s" % commit):
 371        if not foundTitle:
 372            if len(log) == 1:
 373                foundTitle = True
 374            continue
 375
 376        logMessage += log
 377     return logMessage
 378
 379 def extractSettingsGitLog(log):
 380     values = {}
 381     for line in log.split("\n"):
 382         line = line.strip()
 383         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 384         if not m:
 385             continue
 386
 387         assignments = m.group(1).split (':')
 388         for a in assignments:
 389             vals = a.split ('=')
 390             key = vals[0].strip()
 391             val = ('='.join (vals[1:])).strip()
 392             if val.endswith ('\"') and val.startswith('"'):
 393                 val = val[1:-1]
 394
 395             values[key] = val
 396
 397     paths = values.get("depot-paths")
 398     if not paths:
 399         paths = values.get("depot-path")
 400     if paths:
 401         values['depot-paths'] = paths.split(',')
 402     return values
 403
 404 def gitBranchExists(branch):
 405     proc = subprocess.Popen(["git", "rev-parse", branch],
 406                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 407     return proc.wait() == 0;
 408
 409 _gitConfig = {}
 410 def gitConfig(key, args = None): # set args to "--bool", for instance
 411     if not _gitConfig.has_key(key):
 412         argsFilter = ""
 413         if args != None:
 414             argsFilter = "%s " % args
 415         cmd = "git config %s%s" % (argsFilter, key)
 416         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 417     return _gitConfig[key]
 418
 419 def gitConfigList(key):
 420     if not _gitConfig.has_key(key):
 421         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 422     return _gitConfig[key]
 423
 424 def p4BranchesInGit(branchesAreInRemotes = True):
 425     branches = {}
 426
 427     cmdline = "git rev-parse --symbolic "
 428     if branchesAreInRemotes:
 429         cmdline += " --remotes"
 430     else:
 431         cmdline += " --branches"
 432
 433     for line in read_pipe_lines(cmdline):
 434         line = line.strip()
 435
 436         ## only import to p4/
 437         if not line.startswith('p4/') or line == "p4/HEAD":
 438             continue
 439         branch = line
 440
 441         # strip off p4
 442         branch = re.sub ("^p4/", "", line)
 443
 444         branches[branch] = parseRevision(line)
 445     return branches
 446
 447 def findUpstreamBranchPoint(head = "HEAD"):
 448     branches = p4BranchesInGit()
 449     # map from depot-path to branch name
 450     branchByDepotPath = {}
 451     for branch in branches.keys():
 452         tip = branches[branch]
 453         log = extractLogMessageFromGitCommit(tip)
 454         settings = extractSettingsGitLog(log)
 455         if settings.has_key("depot-paths"):
 456             paths = ",".join(settings["depot-paths"])
 457             branchByDepotPath[paths] = "remotes/p4/" + branch
 458
 459     settings = None
 460     parent = 0
 461     while parent < 65535:
 462         commit = head + "~%s" % parent
 463         log = extractLogMessageFromGitCommit(commit)
 464         settings = extractSettingsGitLog(log)
 465         if settings.has_key("depot-paths"):
 466             paths = ",".join(settings["depot-paths"])
 467             if branchByDepotPath.has_key(paths):
 468                 return [branchByDepotPath[paths], settings]
 469
 470         parent = parent + 1
 471
 472     return ["", settings]
 473
 474 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 475     if not silent:
 476         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 477                % localRefPrefix)
 478
 479     originPrefix = "origin/p4/"
 480
 481     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 482         line = line.strip()
 483         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 484             continue
 485
 486         headName = line[len(originPrefix):]
 487         remoteHead = localRefPrefix + headName
 488         originHead = line
 489
 490         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 491         if (not original.has_key('depot-paths')
 492             or not original.has_key('change')):
 493             continue
 494
 495         update = False
 496         if not gitBranchExists(remoteHead):
 497             if verbose:
 498                 print "creating %s" % remoteHead
 499             update = True
 500         else:
 501             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 502             if settings.has_key('change') > 0:
 503                 if settings['depot-paths'] == original['depot-paths']:
 504                     originP4Change = int(original['change'])
 505                     p4Change = int(settings['change'])
 506                     if originP4Change > p4Change:
 507                         print ("%s (%s) is newer than %s (%s). "
 508                                "Updating p4 branch from origin."
 509                                % (originHead, originP4Change,
 510                                   remoteHead, p4Change))
 511                         update = True
 512                 else:
 513                     print ("Ignoring: %s was imported from %s while "
 514                            "%s was imported from %s"
 515                            % (originHead, ','.join(original['depot-paths']),
 516                               remoteHead, ','.join(settings['depot-paths'])))
 517
 518         if update:
 519             system("git update-ref %s %s" % (remoteHead, originHead))
 520
 521 def originP4BranchesExist():
 522         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 523
 524 def p4ChangesForPaths(depotPaths, changeRange):
 525     assert depotPaths
 526     cmd = ['changes']
 527     for p in depotPaths:
 528         cmd += ["%s...%s" % (p, changeRange)]
 529     output = p4_read_pipe_lines(cmd)
 530
 531     changes = {}
 532     for line in output:
 533         changeNum = int(line.split(" ")[1])
 534         changes[changeNum] = True
 535
 536     changelist = changes.keys()
 537     changelist.sort()
 538     return changelist
 539
 540 def p4PathStartsWith(path, prefix):
 541     # This method tries to remedy a potential mixed-case issue:
 542     #
 543     # If UserA adds  //depot/DirA/file1
 544     # and UserB adds //depot/dira/file2
 545     #
 546     # we may or may not have a problem. If you have core.ignorecase=true,
 547     # we treat DirA and dira as the same directory
 548     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 549     if ignorecase:
 550         return path.lower().startswith(prefix.lower())
 551     return path.startswith(prefix)
 552
 553 class Command:
 554     def __init__(self):
 555         self.usage = "usage: %prog [options]"
 556         self.needsGit = True
 557
 558 class P4UserMap:
 559     def __init__(self):
 560         self.userMapFromPerforceServer = False
 561
 562     def getUserCacheFilename(self):
 563         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 564         return home + "/.gitp4-usercache.txt"
 565
 566     def getUserMapFromPerforceServer(self):
 567         if self.userMapFromPerforceServer:
 568             return
 569         self.users = {}
 570         self.emails = {}
 571
 572         for output in p4CmdList("users"):
 573             if not output.has_key("User"):
 574                 continue
 575             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 576             self.emails[output["Email"]] = output["User"]
 577
 578
 579         s = ''
 580         for (key, val) in self.users.items():
 581             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 582
 583         open(self.getUserCacheFilename(), "wb").write(s)
 584         self.userMapFromPerforceServer = True
 585
 586     def loadUserMapFromCache(self):
 587         self.users = {}
 588         self.userMapFromPerforceServer = False
 589         try:
 590             cache = open(self.getUserCacheFilename(), "rb")
 591             lines = cache.readlines()
 592             cache.close()
 593             for line in lines:
 594                 entry = line.strip().split("\t")
 595                 self.users[entry[0]] = entry[1]
 596         except IOError:
 597             self.getUserMapFromPerforceServer()
 598
 599 class P4Debug(Command):
 600     def __init__(self):
 601         Command.__init__(self)
 602         self.options = [
 603             optparse.make_option("--verbose", dest="verbose", action="store_true",
 604                                  default=False),
 605             ]
 606         self.description = "A tool to debug the output of p4 -G."
 607         self.needsGit = False
 608         self.verbose = False
 609
 610     def run(self, args):
 611         j = 0
 612         for output in p4CmdList(args):
 613             print 'Element: %d' % j
 614             j += 1
 615             print output
 616         return True
 617
 618 class P4RollBack(Command):
 619     def __init__(self):
 620         Command.__init__(self)
 621         self.options = [
 622             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 623             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 624         ]
 625         self.description = "A tool to debug the multi-branch import. Don't use :)"
 626         self.verbose = False
 627         self.rollbackLocalBranches = False
 628
 629     def run(self, args):
 630         if len(args) != 1:
 631             return False
 632         maxChange = int(args[0])
 633
 634         if "p4ExitCode" in p4Cmd("changes -m 1"):
 635             die("Problems executing p4");
 636
 637         if self.rollbackLocalBranches:
 638             refPrefix = "refs/heads/"
 639             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 640         else:
 641             refPrefix = "refs/remotes/"
 642             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 643
 644         for line in lines:
 645             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 646                 line = line.strip()
 647                 ref = refPrefix + line
 648                 log = extractLogMessageFromGitCommit(ref)
 649                 settings = extractSettingsGitLog(log)
 650
 651                 depotPaths = settings['depot-paths']
 652                 change = settings['change']
 653
 654                 changed = False
 655
 656                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 657                                                            for p in depotPaths]))) == 0:
 658                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 659                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 660                     continue
 661
 662                 while change and int(change) > maxChange:
 663                     changed = True
 664                     if self.verbose:
 665                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 666                     system("git update-ref %s \"%s^\"" % (ref, ref))
 667                     log = extractLogMessageFromGitCommit(ref)
 668                     settings =  extractSettingsGitLog(log)
 669
 670
 671                     depotPaths = settings['depot-paths']
 672                     change = settings['change']
 673
 674                 if changed:
 675                     print "%s rewound to %s" % (ref, change)
 676
 677         return True
 678
 679 class P4Submit(Command, P4UserMap):
 680     def __init__(self):
 681         Command.__init__(self)
 682         P4UserMap.__init__(self)
 683         self.options = [
 684                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 685                 optparse.make_option("--origin", dest="origin"),
 686                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 687                 # preserve the user, requires relevant p4 permissions
 688                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 689         ]
 690         self.description = "Submit changes from git to the perforce depot."
 691         self.usage += " [name of git branch to submit into perforce depot]"
 692         self.interactive = True
 693         self.origin = ""
 694         self.detectRenames = False
 695         self.verbose = False
 696         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 697         self.isWindows = (platform.system() == "Windows")
 698         self.myP4UserId = None
 699
 700     def check(self):
 701         if len(p4CmdList("opened ...")) > 0:
 702             die("You have files opened with perforce! Close them before starting the sync.")
 703
 704     # replaces everything between 'Description:' and the next P4 submit template field with the
 705     # commit message
 706     def prepareLogMessage(self, template, message):
 707         result = ""
 708
 709         inDescriptionSection = False
 710
 711         for line in template.split("\n"):
 712             if line.startswith("#"):
 713                 result += line + "\n"
 714                 continue
 715
 716             if inDescriptionSection:
 717                 if line.startswith("Files:") or line.startswith("Jobs:"):
 718                     inDescriptionSection = False
 719                 else:
 720                     continue
 721             else:
 722                 if line.startswith("Description:"):
 723                     inDescriptionSection = True
 724                     line += "\n"
 725                     for messageLine in message.split("\n"):
 726                         line += "\t" + messageLine + "\n"
 727
 728             result += line + "\n"
 729
 730         return result
 731
 732     def p4UserForCommit(self,id):
 733         # Return the tuple (perforce user,git email) for a given git commit id
 734         self.getUserMapFromPerforceServer()
 735         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 736         gitEmail = gitEmail.strip()
 737         if not self.emails.has_key(gitEmail):
 738             return (None,gitEmail)
 739         else:
 740             return (self.emails[gitEmail],gitEmail)
 741
 742     def checkValidP4Users(self,commits):
 743         # check if any git authors cannot be mapped to p4 users
 744         for id in commits:
 745             (user,email) = self.p4UserForCommit(id)
 746             if not user:
 747                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 748                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 749                     print "%s" % msg
 750                 else:
 751                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 752
 753     def lastP4Changelist(self):
 754         # Get back the last changelist number submitted in this client spec. This
 755         # then gets used to patch up the username in the change. If the same
 756         # client spec is being used by multiple processes then this might go
 757         # wrong.
 758         results = p4CmdList("client -o")        # find the current client
 759         client = None
 760         for r in results:
 761             if r.has_key('Client'):
 762                 client = r['Client']
 763                 break
 764         if not client:
 765             die("could not get client spec")
 766         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 767         for r in results:
 768             if r.has_key('change'):
 769                 return r['change']
 770         die("Could not get changelist number for last submit - cannot patch up user details")
 771
 772     def modifyChangelistUser(self, changelist, newUser):
 773         # fixup the user field of a changelist after it has been submitted.
 774         changes = p4CmdList("change -o %s" % changelist)
 775         if len(changes) != 1:
 776             die("Bad output from p4 change modifying %s to user %s" %
 777                 (changelist, newUser))
 778
 779         c = changes[0]
 780         if c['User'] == newUser: return   # nothing to do
 781         c['User'] = newUser
 782         input = marshal.dumps(c)
 783
 784         result = p4CmdList("change -f -i", stdin=input)
 785         for r in result:
 786             if r.has_key('code'):
 787                 if r['code'] == 'error':
 788                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 789             if r.has_key('data'):
 790                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 791                 return
 792         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 793
 794     def canChangeChangelists(self):
 795         # check to see if we have p4 admin or super-user permissions, either of
 796         # which are required to modify changelists.
 797         results = p4CmdList("protects %s" % self.depotPath)
 798         for r in results:
 799             if r.has_key('perm'):
 800                 if r['perm'] == 'admin':
 801                     return 1
 802                 if r['perm'] == 'super':
 803                     return 1
 804         return 0
 805
 806     def p4UserId(self):
 807         if self.myP4UserId:
 808             return self.myP4UserId
 809
 810         results = p4CmdList("user -o")
 811         for r in results:
 812             if r.has_key('User'):
 813                 self.myP4UserId = r['User']
 814                 return r['User']
 815         die("Could not find your p4 user id")
 816
 817     def p4UserIsMe(self, p4User):
 818         # return True if the given p4 user is actually me
 819         me = self.p4UserId()
 820         if not p4User or p4User != me:
 821             return False
 822         else:
 823             return True
 824
 825     def prepareSubmitTemplate(self):
 826         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 827         template = ""
 828         inFilesSection = False
 829         for line in p4_read_pipe_lines(['change', '-o']):
 830             if line.endswith("\r\n"):
 831                 line = line[:-2] + "\n"
 832             if inFilesSection:
 833                 if line.startswith("\t"):
 834                     # path starts and ends with a tab
 835                     path = line[1:]
 836                     lastTab = path.rfind("\t")
 837                     if lastTab != -1:
 838                         path = path[:lastTab]
 839                         if not p4PathStartsWith(path, self.depotPath):
 840                             continue
 841                 else:
 842                     inFilesSection = False
 843             else:
 844                 if line.startswith("Files:"):
 845                     inFilesSection = True
 846
 847             template += line
 848
 849         return template
 850
 851     def edit_template(self, template_file):
 852         """Invoke the editor to let the user change the submission
 853            message.  Return true if okay to continue with the submit."""
 854
 855         # if configured to skip the editing part, just submit
 856         if gitConfig("git-p4.skipSubmitEdit") == "true":
 857             return True
 858
 859         # look at the modification time, to check later if the user saved
 860         # the file
 861         mtime = os.stat(template_file).st_mtime
 862
 863         # invoke the editor
 864         if os.environ.has_key("P4EDITOR"):
 865             editor = os.environ.get("P4EDITOR")
 866         else:
 867             editor = read_pipe("git var GIT_EDITOR").strip()
 868         system(editor + " " + template_file)
 869
 870         # If the file was not saved, prompt to see if this patch should
 871         # be skipped.  But skip this verification step if configured so.
 872         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 873             return True
 874
 875         # modification time updated means user saved the file
 876         if os.stat(template_file).st_mtime > mtime:
 877             return True
 878
 879         while True:
 880             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 881             if response == 'y':
 882                 return True
 883             if response == 'n':
 884                 return False
 885
 886     def applyCommit(self, id):
 887         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 888
 889         (p4User, gitEmail) = self.p4UserForCommit(id)
 890
 891         if not self.detectRenames:
 892             # If not explicitly set check the config variable
 893             self.detectRenames = gitConfig("git-p4.detectRenames")
 894
 895         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 896             diffOpts = ""
 897         elif self.detectRenames.lower() == "true":
 898             diffOpts = "-M"
 899         else:
 900             diffOpts = "-M%s" % self.detectRenames
 901
 902         detectCopies = gitConfig("git-p4.detectCopies")
 903         if detectCopies.lower() == "true":
 904             diffOpts += " -C"
 905         elif detectCopies != "" and detectCopies.lower() != "false":
 906             diffOpts += " -C%s" % detectCopies
 907
 908         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 909             diffOpts += " --find-copies-harder"
 910
 911         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 912         filesToAdd = set()
 913         filesToDelete = set()
 914         editedFiles = set()
 915         filesToChangeExecBit = {}
 916         for line in diff:
 917             diff = parseDiffTreeEntry(line)
 918             modifier = diff['status']
 919             path = diff['src']
 920             if modifier == "M":
 921                 p4_edit(path)
 922                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 923                     filesToChangeExecBit[path] = diff['dst_mode']
 924                 editedFiles.add(path)
 925             elif modifier == "A":
 926                 filesToAdd.add(path)
 927                 filesToChangeExecBit[path] = diff['dst_mode']
 928                 if path in filesToDelete:
 929                     filesToDelete.remove(path)
 930             elif modifier == "D":
 931                 filesToDelete.add(path)
 932                 if path in filesToAdd:
 933                     filesToAdd.remove(path)
 934             elif modifier == "C":
 935                 src, dest = diff['src'], diff['dst']
 936                 p4_integrate(src, dest)
 937                 if diff['src_sha1'] != diff['dst_sha1']:
 938                     p4_edit(dest)
 939                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 940                     p4_edit(dest)
 941                     filesToChangeExecBit[dest] = diff['dst_mode']
 942                 os.unlink(dest)
 943                 editedFiles.add(dest)
 944             elif modifier == "R":
 945                 src, dest = diff['src'], diff['dst']
 946                 p4_integrate(src, dest)
 947                 if diff['src_sha1'] != diff['dst_sha1']:
 948                     p4_edit(dest)
 949                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 950                     p4_edit(dest)
 951                     filesToChangeExecBit[dest] = diff['dst_mode']
 952                 os.unlink(dest)
 953                 editedFiles.add(dest)
 954                 filesToDelete.add(src)
 955             else:
 956                 die("unknown modifier %s for %s" % (modifier, path))
 957
 958         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 959         patchcmd = diffcmd + " | git apply "
 960         tryPatchCmd = patchcmd + "--check -"
 961         applyPatchCmd = patchcmd + "--check --apply -"
 962
 963         if os.system(tryPatchCmd) != 0:
 964             print "Unfortunately applying the change failed!"
 965             print "What do you want to do?"
 966             response = "x"
 967             while response != "s" and response != "a" and response != "w":
 968                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 969                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 970             if response == "s":
 971                 print "Skipping! Good luck with the next patches..."
 972                 for f in editedFiles:
 973                     p4_revert(f)
 974                 for f in filesToAdd:
 975                     os.remove(f)
 976                 return
 977             elif response == "a":
 978                 os.system(applyPatchCmd)
 979                 if len(filesToAdd) > 0:
 980                     print "You may also want to call p4 add on the following files:"
 981                     print " ".join(filesToAdd)
 982                 if len(filesToDelete):
 983                     print "The following files should be scheduled for deletion with p4 delete:"
 984                     print " ".join(filesToDelete)
 985                 die("Please resolve and submit the conflict manually and "
 986                     + "continue afterwards with git-p4 submit --continue")
 987             elif response == "w":
 988                 system(diffcmd + " > patch.txt")
 989                 print "Patch saved to patch.txt in %s !" % self.clientPath
 990                 die("Please resolve and submit the conflict manually and "
 991                     "continue afterwards with git-p4 submit --continue")
 992
 993         system(applyPatchCmd)
 994
 995         for f in filesToAdd:
 996             p4_add(f)
 997         for f in filesToDelete:
 998             p4_revert(f)
 999             p4_delete(f)
1000
1001         # Set/clear executable bits
1002         for f in filesToChangeExecBit.keys():
1003             mode = filesToChangeExecBit[f]
1004             setP4ExecBit(f, mode)
1005
1006         logMessage = extractLogMessageFromGitCommit(id)
1007         logMessage = logMessage.strip()
1008
1009         template = self.prepareSubmitTemplate()
1010
1011         if self.interactive:
1012             submitTemplate = self.prepareLogMessage(template, logMessage)
1013
1014             if self.preserveUser:
1015                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1016
1017             if os.environ.has_key("P4DIFF"):
1018                 del(os.environ["P4DIFF"])
1019             diff = ""
1020             for editedFile in editedFiles:
1021                 diff += p4_read_pipe(['diff', '-du', editedFile])
1022
1023             newdiff = ""
1024             for newFile in filesToAdd:
1025                 newdiff += "==== new file ====\n"
1026                 newdiff += "--- /dev/null\n"
1027                 newdiff += "+++ %s\n" % newFile
1028                 f = open(newFile, "r")
1029                 for line in f.readlines():
1030                     newdiff += "+" + line
1031                 f.close()
1032
1033             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1034                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1035                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1036                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1037
1038             separatorLine = "######## everything below this line is just the diff #######\n"
1039
1040             (handle, fileName) = tempfile.mkstemp()
1041             tmpFile = os.fdopen(handle, "w+")
1042             if self.isWindows:
1043                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1044                 separatorLine = separatorLine.replace("\n", "\r\n")
1045                 newdiff = newdiff.replace("\n", "\r\n")
1046             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1047             tmpFile.close()
1048
1049             if self.edit_template(fileName):
1050                 # read the edited message and submit
1051                 tmpFile = open(fileName, "rb")
1052                 message = tmpFile.read()
1053                 tmpFile.close()
1054                 submitTemplate = message[:message.index(separatorLine)]
1055                 if self.isWindows:
1056                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1057                 p4_write_pipe(['submit', '-i'], submitTemplate)
1058
1059                 if self.preserveUser:
1060                     if p4User:
1061                         # Get last changelist number. Cannot easily get it from
1062                         # the submit command output as the output is
1063                         # unmarshalled.
1064                         changelist = self.lastP4Changelist()
1065                         self.modifyChangelistUser(changelist, p4User)
1066             else:
1067                 # skip this patch
1068                 print "Submission cancelled, undoing p4 changes."
1069                 for f in editedFiles:
1070                     p4_revert(f)
1071                 for f in filesToAdd:
1072                     p4_revert(f)
1073                     os.remove(f)
1074
1075             os.remove(fileName)
1076         else:
1077             fileName = "submit.txt"
1078             file = open(fileName, "w+")
1079             file.write(self.prepareLogMessage(template, logMessage))
1080             file.close()
1081             print ("Perforce submit template written as %s. "
1082                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1083                    % (fileName, fileName))
1084
1085     def run(self, args):
1086         if len(args) == 0:
1087             self.master = currentGitBranch()
1088             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1089                 die("Detecting current git branch failed!")
1090         elif len(args) == 1:
1091             self.master = args[0]
1092         else:
1093             return False
1094
1095         allowSubmit = gitConfig("git-p4.allowSubmit")
1096         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1097             die("%s is not in git-p4.allowSubmit" % self.master)
1098
1099         [upstream, settings] = findUpstreamBranchPoint()
1100         self.depotPath = settings['depot-paths'][0]
1101         if len(self.origin) == 0:
1102             self.origin = upstream
1103
1104         if self.preserveUser:
1105             if not self.canChangeChangelists():
1106                 die("Cannot preserve user names without p4 super-user or admin permissions")
1107
1108         if self.verbose:
1109             print "Origin branch is " + self.origin
1110
1111         if len(self.depotPath) == 0:
1112             print "Internal error: cannot locate perforce depot path from existing branches"
1113             sys.exit(128)
1114
1115         self.clientPath = p4Where(self.depotPath)
1116
1117         if len(self.clientPath) == 0:
1118             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1119             sys.exit(128)
1120
1121         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1122         self.oldWorkingDirectory = os.getcwd()
1123
1124         # ensure the clientPath exists
1125         if not os.path.exists(self.clientPath):
1126             os.makedirs(self.clientPath)
1127
1128         chdir(self.clientPath)
1129         print "Synchronizing p4 checkout..."
1130         p4_sync("...")
1131         self.check()
1132
1133         commits = []
1134         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1135             commits.append(line.strip())
1136         commits.reverse()
1137
1138         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1139             self.checkAuthorship = False
1140         else:
1141             self.checkAuthorship = True
1142
1143         if self.preserveUser:
1144             self.checkValidP4Users(commits)
1145
1146         while len(commits) > 0:
1147             commit = commits[0]
1148             commits = commits[1:]
1149             self.applyCommit(commit)
1150             if not self.interactive:
1151                 break
1152
1153         if len(commits) == 0:
1154             print "All changes applied!"
1155             chdir(self.oldWorkingDirectory)
1156
1157             sync = P4Sync()
1158             sync.run([])
1159
1160             rebase = P4Rebase()
1161             rebase.rebase()
1162
1163         return True
1164
1165 class P4Sync(Command, P4UserMap):
1166     delete_actions = ( "delete", "move/delete", "purge" )
1167
1168     def __init__(self):
1169         Command.__init__(self)
1170         P4UserMap.__init__(self)
1171         self.options = [
1172                 optparse.make_option("--branch", dest="branch"),
1173                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1174                 optparse.make_option("--changesfile", dest="changesFile"),
1175                 optparse.make_option("--silent", dest="silent", action="store_true"),
1176                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1177                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1178                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1179                                      help="Import into refs/heads/ , not refs/remotes"),
1180                 optparse.make_option("--max-changes", dest="maxChanges"),
1181                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1182                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1183                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1184                                      help="Only sync files that are included in the Perforce Client Spec")
1185         ]
1186         self.description = """Imports from Perforce into a git repository.\n
1187     example:
1188     //depot/my/project/ -- to import the current head
1189     //depot/my/project/@all -- to import everything
1190     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1191
1192     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1193
1194         self.usage += " //depot/path[@revRange]"
1195         self.silent = False
1196         self.createdBranches = set()
1197         self.committedChanges = set()
1198         self.branch = ""
1199         self.detectBranches = False
1200         self.detectLabels = False
1201         self.changesFile = ""
1202         self.syncWithOrigin = True
1203         self.verbose = False
1204         self.importIntoRemotes = True
1205         self.maxChanges = ""
1206         self.isWindows = (platform.system() == "Windows")
1207         self.keepRepoPath = False
1208         self.depotPaths = None
1209         self.p4BranchesInGit = []
1210         self.cloneExclude = []
1211         self.useClientSpec = False
1212         self.clientSpecDirs = []
1213
1214         if gitConfig("git-p4.syncFromOrigin") == "false":
1215             self.syncWithOrigin = False
1216
1217     #
1218     # P4 wildcards are not allowed in filenames.  P4 complains
1219     # if you simply add them, but you can force it with "-f", in
1220     # which case it translates them into %xx encoding internally.
1221     # Search for and fix just these four characters.  Do % last so
1222     # that fixing it does not inadvertently create new %-escapes.
1223     #
1224     def wildcard_decode(self, path):
1225         # Cannot have * in a filename in windows; untested as to
1226         # what p4 would do in such a case.
1227         if not self.isWindows:
1228             path = path.replace("%2A", "*")
1229         path = path.replace("%23", "#") \
1230                    .replace("%40", "@") \
1231                    .replace("%25", "%")
1232         return path
1233
1234     def extractFilesFromCommit(self, commit):
1235         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1236                              for path in self.cloneExclude]
1237         files = []
1238         fnum = 0
1239         while commit.has_key("depotFile%s" % fnum):
1240             path =  commit["depotFile%s" % fnum]
1241
1242             if [p for p in self.cloneExclude
1243                 if p4PathStartsWith(path, p)]:
1244                 found = False
1245             else:
1246                 found = [p for p in self.depotPaths
1247                          if p4PathStartsWith(path, p)]
1248             if not found:
1249                 fnum = fnum + 1
1250                 continue
1251
1252             file = {}
1253             file["path"] = path
1254             file["rev"] = commit["rev%s" % fnum]
1255             file["action"] = commit["action%s" % fnum]
1256             file["type"] = commit["type%s" % fnum]
1257             files.append(file)
1258             fnum = fnum + 1
1259         return files
1260
1261     def stripRepoPath(self, path, prefixes):
1262         if self.useClientSpec:
1263
1264             # if using the client spec, we use the output directory
1265             # specified in the client.  For example, a view
1266             #   //depot/foo/branch/... //client/branch/foo/...
1267             # will end up putting all foo/branch files into
1268             #  branch/foo/
1269             for val in self.clientSpecDirs:
1270                 if path.startswith(val[0]):
1271                     # replace the depot path with the client path
1272                     path = path.replace(val[0], val[1][1])
1273                     # now strip out the client (//client/...)
1274                     path = re.sub("^(//[^/]+/)", '', path)
1275                     # the rest is all path
1276                     return path
1277
1278         if self.keepRepoPath:
1279             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1280
1281         for p in prefixes:
1282             if p4PathStartsWith(path, p):
1283                 path = path[len(p):]
1284
1285         return path
1286
1287     def splitFilesIntoBranches(self, commit):
1288         branches = {}
1289         fnum = 0
1290         while commit.has_key("depotFile%s" % fnum):
1291             path =  commit["depotFile%s" % fnum]
1292             found = [p for p in self.depotPaths
1293                      if p4PathStartsWith(path, p)]
1294             if not found:
1295                 fnum = fnum + 1
1296                 continue
1297
1298             file = {}
1299             file["path"] = path
1300             file["rev"] = commit["rev%s" % fnum]
1301             file["action"] = commit["action%s" % fnum]
1302             file["type"] = commit["type%s" % fnum]
1303             fnum = fnum + 1
1304
1305             relPath = self.stripRepoPath(path, self.depotPaths)
1306
1307             for branch in self.knownBranches.keys():
1308
1309                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1310                 if relPath.startswith(branch + "/"):
1311                     if branch not in branches:
1312                         branches[branch] = []
1313                     branches[branch].append(file)
1314                     break
1315
1316         return branches
1317
1318     # output one file from the P4 stream
1319     # - helper for streamP4Files
1320
1321     def streamOneP4File(self, file, contents):
1322         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1323         relPath = self.wildcard_decode(relPath)
1324         if verbose:
1325             sys.stderr.write("%s\n" % relPath)
1326
1327         (type_base, type_mods) = split_p4_type(file["type"])
1328
1329         git_mode = "100644"
1330         if "x" in type_mods:
1331             git_mode = "100755"
1332         if type_base == "symlink":
1333             git_mode = "120000"
1334             # p4 print on a symlink contains "target\n"; remove the newline
1335             data = ''.join(contents)
1336             contents = [data[:-1]]
1337
1338         if type_base == "utf16":
1339             # p4 delivers different text in the python output to -G
1340             # than it does when using "print -o", or normal p4 client
1341             # operations.  utf16 is converted to ascii or utf8, perhaps.
1342             # But ascii text saved as -t utf16 is completely mangled.
1343             # Invoke print -o to get the real contents.
1344             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1345             contents = [ text ]
1346
1347         if type_base == "apple":
1348             # Apple filetype files will be streamed as a concatenation of
1349             # its appledouble header and the contents.  This is useless
1350             # on both macs and non-macs.  If using "print -q -o xx", it
1351             # will create "xx" with the data, and "%xx" with the header.
1352             # This is also not very useful.
1353             #
1354             # Ideally, someday, this script can learn how to generate
1355             # appledouble files directly and import those to git, but
1356             # non-mac machines can never find a use for apple filetype.
1357             print "\nIgnoring apple filetype file %s" % file['depotFile']
1358             return
1359
1360         # Perhaps windows wants unicode, utf16 newlines translated too;
1361         # but this is not doing it.
1362         if self.isWindows and type_base == "text":
1363             mangled = []
1364             for data in contents:
1365                 data = data.replace("\r\n", "\n")
1366                 mangled.append(data)
1367             contents = mangled
1368
1369         # Note that we do not try to de-mangle keywords on utf16 files,
1370         # even though in theory somebody may want that.
1371         if type_base in ("text", "unicode", "binary"):
1372             if "ko" in type_mods:
1373                 text = ''.join(contents)
1374                 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1375                 contents = [ text ]
1376             elif "k" in type_mods:
1377                 text = ''.join(contents)
1378                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1379                 contents = [ text ]
1380
1381         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1382
1383         # total length...
1384         length = 0
1385         for d in contents:
1386             length = length + len(d)
1387
1388         self.gitStream.write("data %d\n" % length)
1389         for d in contents:
1390             self.gitStream.write(d)
1391         self.gitStream.write("\n")
1392
1393     def streamOneP4Deletion(self, file):
1394         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1395         if verbose:
1396             sys.stderr.write("delete %s\n" % relPath)
1397         self.gitStream.write("D %s\n" % relPath)
1398
1399     # handle another chunk of streaming data
1400     def streamP4FilesCb(self, marshalled):
1401
1402         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1403             # start of a new file - output the old one first
1404             self.streamOneP4File(self.stream_file, self.stream_contents)
1405             self.stream_file = {}
1406             self.stream_contents = []
1407             self.stream_have_file_info = False
1408
1409         # pick up the new file information... for the
1410         # 'data' field we need to append to our array
1411         for k in marshalled.keys():
1412             if k == 'data':
1413                 self.stream_contents.append(marshalled['data'])
1414             else:
1415                 self.stream_file[k] = marshalled[k]
1416
1417         self.stream_have_file_info = True
1418
1419     # Stream directly from "p4 files" into "git fast-import"
1420     def streamP4Files(self, files):
1421         filesForCommit = []
1422         filesToRead = []
1423         filesToDelete = []
1424
1425         for f in files:
1426             includeFile = True
1427             for val in self.clientSpecDirs:
1428                 if f['path'].startswith(val[0]):
1429                     if val[1][0] <= 0:
1430                         includeFile = False
1431                     break
1432
1433             if includeFile:
1434                 filesForCommit.append(f)
1435                 if f['action'] in self.delete_actions:
1436                     filesToDelete.append(f)
1437                 else:
1438                     filesToRead.append(f)
1439
1440         # deleted files...
1441         for f in filesToDelete:
1442             self.streamOneP4Deletion(f)
1443
1444         if len(filesToRead) > 0:
1445             self.stream_file = {}
1446             self.stream_contents = []
1447             self.stream_have_file_info = False
1448
1449             # curry self argument
1450             def streamP4FilesCbSelf(entry):
1451                 self.streamP4FilesCb(entry)
1452
1453             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1454
1455             p4CmdList(["-x", "-", "print"],
1456                       stdin=fileArgs,
1457                       cb=streamP4FilesCbSelf)
1458
1459             # do the last chunk
1460             if self.stream_file.has_key('depotFile'):
1461                 self.streamOneP4File(self.stream_file, self.stream_contents)
1462
1463     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1464         epoch = details["time"]
1465         author = details["user"]
1466         self.branchPrefixes = branchPrefixes
1467
1468         if self.verbose:
1469             print "commit into %s" % branch
1470
1471         # start with reading files; if that fails, we should not
1472         # create a commit.
1473         new_files = []
1474         for f in files:
1475             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1476                 new_files.append (f)
1477             else:
1478                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1479
1480         self.gitStream.write("commit %s\n" % branch)
1481 #        gitStream.write("mark :%s\n" % details["change"])
1482         self.committedChanges.add(int(details["change"]))
1483         committer = ""
1484         if author not in self.users:
1485             self.getUserMapFromPerforceServer()
1486         if author in self.users:
1487             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1488         else:
1489             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1490
1491         self.gitStream.write("committer %s\n" % committer)
1492
1493         self.gitStream.write("data <<EOT\n")
1494         self.gitStream.write(details["desc"])
1495         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1496                              % (','.join (branchPrefixes), details["change"]))
1497         if len(details['options']) > 0:
1498             self.gitStream.write(": options = %s" % details['options'])
1499         self.gitStream.write("]\nEOT\n\n")
1500
1501         if len(parent) > 0:
1502             if self.verbose:
1503                 print "parent %s" % parent
1504             self.gitStream.write("from %s\n" % parent)
1505
1506         self.streamP4Files(new_files)
1507         self.gitStream.write("\n")
1508
1509         change = int(details["change"])
1510
1511         if self.labels.has_key(change):
1512             label = self.labels[change]
1513             labelDetails = label[0]
1514             labelRevisions = label[1]
1515             if self.verbose:
1516                 print "Change %s is labelled %s" % (change, labelDetails)
1517
1518             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1519                                                     for p in branchPrefixes])
1520
1521             if len(files) == len(labelRevisions):
1522
1523                 cleanedFiles = {}
1524                 for info in files:
1525                     if info["action"] in self.delete_actions:
1526                         continue
1527                     cleanedFiles[info["depotFile"]] = info["rev"]
1528
1529                 if cleanedFiles == labelRevisions:
1530                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1531                     self.gitStream.write("from %s\n" % branch)
1532
1533                     owner = labelDetails["Owner"]
1534                     tagger = ""
1535                     if author in self.users:
1536                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1537                     else:
1538                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1539                     self.gitStream.write("tagger %s\n" % tagger)
1540                     self.gitStream.write("data <<EOT\n")
1541                     self.gitStream.write(labelDetails["Description"])
1542                     self.gitStream.write("EOT\n\n")
1543
1544                 else:
1545                     if not self.silent:
1546                         print ("Tag %s does not match with change %s: files do not match."
1547                                % (labelDetails["label"], change))
1548
1549             else:
1550                 if not self.silent:
1551                     print ("Tag %s does not match with change %s: file count is different."
1552                            % (labelDetails["label"], change))
1553
1554     def getLabels(self):
1555         self.labels = {}
1556
1557         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1558         if len(l) > 0 and not self.silent:
1559             print "Finding files belonging to labels in %s" % `self.depotPaths`
1560
1561         for output in l:
1562             label = output["label"]
1563             revisions = {}
1564             newestChange = 0
1565             if self.verbose:
1566                 print "Querying files for label %s" % label
1567             for file in p4CmdList(["files"] +
1568                                       ["%s...@%s" % (p, label)
1569                                           for p in self.depotPaths]):
1570                 revisions[file["depotFile"]] = file["rev"]
1571                 change = int(file["change"])
1572                 if change > newestChange:
1573                     newestChange = change
1574
1575             self.labels[newestChange] = [output, revisions]
1576
1577         if self.verbose:
1578             print "Label changes: %s" % self.labels.keys()
1579
1580     def guessProjectName(self):
1581         for p in self.depotPaths:
1582             if p.endswith("/"):
1583                 p = p[:-1]
1584             p = p[p.strip().rfind("/") + 1:]
1585             if not p.endswith("/"):
1586                p += "/"
1587             return p
1588
1589     def getBranchMapping(self):
1590         lostAndFoundBranches = set()
1591
1592         user = gitConfig("git-p4.branchUser")
1593         if len(user) > 0:
1594             command = "branches -u %s" % user
1595         else:
1596             command = "branches"
1597
1598         for info in p4CmdList(command):
1599             details = p4Cmd("branch -o %s" % info["branch"])
1600             viewIdx = 0
1601             while details.has_key("View%s" % viewIdx):
1602                 paths = details["View%s" % viewIdx].split(" ")
1603                 viewIdx = viewIdx + 1
1604                 # require standard //depot/foo/... //depot/bar/... mapping
1605                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1606                     continue
1607                 source = paths[0]
1608                 destination = paths[1]
1609                 ## HACK
1610                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1611                     source = source[len(self.depotPaths[0]):-4]
1612                     destination = destination[len(self.depotPaths[0]):-4]
1613
1614                     if destination in self.knownBranches:
1615                         if not self.silent:
1616                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1617                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1618                         continue
1619
1620                     self.knownBranches[destination] = source
1621
1622                     lostAndFoundBranches.discard(destination)
1623
1624                     if source not in self.knownBranches:
1625                         lostAndFoundBranches.add(source)
1626
1627         # Perforce does not strictly require branches to be defined, so we also
1628         # check git config for a branch list.
1629         #
1630         # Example of branch definition in git config file:
1631         # [git-p4]
1632         #   branchList=main:branchA
1633         #   branchList=main:branchB
1634         #   branchList=branchA:branchC
1635         configBranches = gitConfigList("git-p4.branchList")
1636         for branch in configBranches:
1637             if branch:
1638                 (source, destination) = branch.split(":")
1639                 self.knownBranches[destination] = source
1640
1641                 lostAndFoundBranches.discard(destination)
1642
1643                 if source not in self.knownBranches:
1644                     lostAndFoundBranches.add(source)
1645
1646
1647         for branch in lostAndFoundBranches:
1648             self.knownBranches[branch] = branch
1649
1650     def getBranchMappingFromGitBranches(self):
1651         branches = p4BranchesInGit(self.importIntoRemotes)
1652         for branch in branches.keys():
1653             if branch == "master":
1654                 branch = "main"
1655             else:
1656                 branch = branch[len(self.projectName):]
1657             self.knownBranches[branch] = branch
1658
1659     def listExistingP4GitBranches(self):
1660         # branches holds mapping from name to commit
1661         branches = p4BranchesInGit(self.importIntoRemotes)
1662         self.p4BranchesInGit = branches.keys()
1663         for branch in branches.keys():
1664             self.initialParents[self.refPrefix + branch] = branches[branch]
1665
1666     def updateOptionDict(self, d):
1667         option_keys = {}
1668         if self.keepRepoPath:
1669             option_keys['keepRepoPath'] = 1
1670
1671         d["options"] = ' '.join(sorted(option_keys.keys()))
1672
1673     def readOptions(self, d):
1674         self.keepRepoPath = (d.has_key('options')
1675                              and ('keepRepoPath' in d['options']))
1676
1677     def gitRefForBranch(self, branch):
1678         if branch == "main":
1679             return self.refPrefix + "master"
1680
1681         if len(branch) <= 0:
1682             return branch
1683
1684         return self.refPrefix + self.projectName + branch
1685
1686     def gitCommitByP4Change(self, ref, change):
1687         if self.verbose:
1688             print "looking in ref " + ref + " for change %s using bisect..." % change
1689
1690         earliestCommit = ""
1691         latestCommit = parseRevision(ref)
1692
1693         while True:
1694             if self.verbose:
1695                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1696             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1697             if len(next) == 0:
1698                 if self.verbose:
1699                     print "argh"
1700                 return ""
1701             log = extractLogMessageFromGitCommit(next)
1702             settings = extractSettingsGitLog(log)
1703             currentChange = int(settings['change'])
1704             if self.verbose:
1705                 print "current change %s" % currentChange
1706
1707             if currentChange == change:
1708                 if self.verbose:
1709                     print "found %s" % next
1710                 return next
1711
1712             if currentChange < change:
1713                 earliestCommit = "^%s" % next
1714             else:
1715                 latestCommit = "%s" % next
1716
1717         return ""
1718
1719     def importNewBranch(self, branch, maxChange):
1720         # make fast-import flush all changes to disk and update the refs using the checkpoint
1721         # command so that we can try to find the branch parent in the git history
1722         self.gitStream.write("checkpoint\n\n");
1723         self.gitStream.flush();
1724         branchPrefix = self.depotPaths[0] + branch + "/"
1725         range = "@1,%s" % maxChange
1726         #print "prefix" + branchPrefix
1727         changes = p4ChangesForPaths([branchPrefix], range)
1728         if len(changes) <= 0:
1729             return False
1730         firstChange = changes[0]
1731         #print "first change in branch: %s" % firstChange
1732         sourceBranch = self.knownBranches[branch]
1733         sourceDepotPath = self.depotPaths[0] + sourceBranch
1734         sourceRef = self.gitRefForBranch(sourceBranch)
1735         #print "source " + sourceBranch
1736
1737         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1738         #print "branch parent: %s" % branchParentChange
1739         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1740         if len(gitParent) > 0:
1741             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1742             #print "parent git commit: %s" % gitParent
1743
1744         self.importChanges(changes)
1745         return True
1746
1747     def importChanges(self, changes):
1748         cnt = 1
1749         for change in changes:
1750             description = p4Cmd("describe %s" % change)
1751             self.updateOptionDict(description)
1752
1753             if not self.silent:
1754                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1755                 sys.stdout.flush()
1756             cnt = cnt + 1
1757
1758             try:
1759                 if self.detectBranches:
1760                     branches = self.splitFilesIntoBranches(description)
1761                     for branch in branches.keys():
1762                         ## HACK  --hwn
1763                         branchPrefix = self.depotPaths[0] + branch + "/"
1764
1765                         parent = ""
1766
1767                         filesForCommit = branches[branch]
1768
1769                         if self.verbose:
1770                             print "branch is %s" % branch
1771
1772                         self.updatedBranches.add(branch)
1773
1774                         if branch not in self.createdBranches:
1775                             self.createdBranches.add(branch)
1776                             parent = self.knownBranches[branch]
1777                             if parent == branch:
1778                                 parent = ""
1779                             else:
1780                                 fullBranch = self.projectName + branch
1781                                 if fullBranch not in self.p4BranchesInGit:
1782                                     if not self.silent:
1783                                         print("\n    Importing new branch %s" % fullBranch);
1784                                     if self.importNewBranch(branch, change - 1):
1785                                         parent = ""
1786                                         self.p4BranchesInGit.append(fullBranch)
1787                                     if not self.silent:
1788                                         print("\n    Resuming with change %s" % change);
1789
1790                                 if self.verbose:
1791                                     print "parent determined through known branches: %s" % parent
1792
1793                         branch = self.gitRefForBranch(branch)
1794                         parent = self.gitRefForBranch(parent)
1795
1796                         if self.verbose:
1797                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1798
1799                         if len(parent) == 0 and branch in self.initialParents:
1800                             parent = self.initialParents[branch]
1801                             del self.initialParents[branch]
1802
1803                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1804                 else:
1805                     files = self.extractFilesFromCommit(description)
1806                     self.commit(description, files, self.branch, self.depotPaths,
1807                                 self.initialParent)
1808                     self.initialParent = ""
1809             except IOError:
1810                 print self.gitError.read()
1811                 sys.exit(1)
1812
1813     def importHeadRevision(self, revision):
1814         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1815
1816         details = {}
1817         details["user"] = "git perforce import user"
1818         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1819                            % (' '.join(self.depotPaths), revision))
1820         details["change"] = revision
1821         newestRevision = 0
1822
1823         fileCnt = 0
1824         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
1825
1826         for info in p4CmdList(["files"] + fileArgs):
1827
1828             if 'code' in info and info['code'] == 'error':
1829                 sys.stderr.write("p4 returned an error: %s\n"
1830                                  % info['data'])
1831                 if info['data'].find("must refer to client") >= 0:
1832                     sys.stderr.write("This particular p4 error is misleading.\n")
1833                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1834                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1835                 sys.exit(1)
1836             if 'p4ExitCode' in info:
1837                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1838                 sys.exit(1)
1839
1840
1841             change = int(info["change"])
1842             if change > newestRevision:
1843                 newestRevision = change
1844
1845             if info["action"] in self.delete_actions:
1846                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1847                 #fileCnt = fileCnt + 1
1848                 continue
1849
1850             for prop in ["depotFile", "rev", "action", "type" ]:
1851                 details["%s%s" % (prop, fileCnt)] = info[prop]
1852
1853             fileCnt = fileCnt + 1
1854
1855         details["change"] = newestRevision
1856
1857         # Use time from top-most change so that all git-p4 clones of
1858         # the same p4 repo have the same commit SHA1s.
1859         res = p4CmdList("describe -s %d" % newestRevision)
1860         newestTime = None
1861         for r in res:
1862             if r.has_key('time'):
1863                 newestTime = int(r['time'])
1864         if newestTime is None:
1865             die("\"describe -s\" on newest change %d did not give a time")
1866         details["time"] = newestTime
1867
1868         self.updateOptionDict(details)
1869         try:
1870             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1871         except IOError:
1872             print "IO error with git fast-import. Is your git version recent enough?"
1873             print self.gitError.read()
1874
1875
1876     def getClientSpec(self):
1877         specList = p4CmdList( "client -o" )
1878         temp = {}
1879         for entry in specList:
1880             for k,v in entry.iteritems():
1881                 if k.startswith("View"):
1882
1883                     # p4 has these %%1 to %%9 arguments in specs to
1884                     # reorder paths; which we can't handle (yet :)
1885                     if re.match('%%\d', v) != None:
1886                         print "Sorry, can't handle %%n arguments in client specs"
1887                         sys.exit(1)
1888
1889                     if v.startswith('"'):
1890                         start = 1
1891                     else:
1892                         start = 0
1893                     index = v.find("...")
1894
1895                     # save the "client view"; i.e the RHS of the view
1896                     # line that tells the client where to put the
1897                     # files for this view.
1898                     cv = v[index+3:].strip() # +3 to remove previous '...'
1899
1900                     # if the client view doesn't end with a
1901                     # ... wildcard, then we're going to mess up the
1902                     # output directory, so fail gracefully.
1903                     if not cv.endswith('...'):
1904                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1905                         sys.exit(1)
1906                     cv=cv[:-3]
1907
1908                     # now save the view; +index means included, -index
1909                     # means it should be filtered out.
1910                     v = v[start:index]
1911                     if v.startswith("-"):
1912                         v = v[1:]
1913                         include = -len(v)
1914                     else:
1915                         include = len(v)
1916
1917                     temp[v] = (include, cv)
1918
1919         self.clientSpecDirs = temp.items()
1920         self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1921
1922     def run(self, args):
1923         self.depotPaths = []
1924         self.changeRange = ""
1925         self.initialParent = ""
1926         self.previousDepotPaths = []
1927
1928         # map from branch depot path to parent branch
1929         self.knownBranches = {}
1930         self.initialParents = {}
1931         self.hasOrigin = originP4BranchesExist()
1932         if not self.syncWithOrigin:
1933             self.hasOrigin = False
1934
1935         if self.importIntoRemotes:
1936             self.refPrefix = "refs/remotes/p4/"
1937         else:
1938             self.refPrefix = "refs/heads/p4/"
1939
1940         if self.syncWithOrigin and self.hasOrigin:
1941             if not self.silent:
1942                 print "Syncing with origin first by calling git fetch origin"
1943             system("git fetch origin")
1944
1945         if len(self.branch) == 0:
1946             self.branch = self.refPrefix + "master"
1947             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1948                 system("git update-ref %s refs/heads/p4" % self.branch)
1949                 system("git branch -D p4");
1950             # create it /after/ importing, when master exists
1951             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1952                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1953
1954         if not self.useClientSpec:
1955             if gitConfig("git-p4.useclientspec", "--bool") == "true":
1956                 self.useClientSpec = True
1957         if self.useClientSpec:
1958             self.getClientSpec()
1959
1960         # TODO: should always look at previous commits,
1961         # merge with previous imports, if possible.
1962         if args == []:
1963             if self.hasOrigin:
1964                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1965             self.listExistingP4GitBranches()
1966
1967             if len(self.p4BranchesInGit) > 1:
1968                 if not self.silent:
1969                     print "Importing from/into multiple branches"
1970                 self.detectBranches = True
1971
1972             if self.verbose:
1973                 print "branches: %s" % self.p4BranchesInGit
1974
1975             p4Change = 0
1976             for branch in self.p4BranchesInGit:
1977                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1978
1979                 settings = extractSettingsGitLog(logMsg)
1980
1981                 self.readOptions(settings)
1982                 if (settings.has_key('depot-paths')
1983                     and settings.has_key ('change')):
1984                     change = int(settings['change']) + 1
1985                     p4Change = max(p4Change, change)
1986
1987                     depotPaths = sorted(settings['depot-paths'])
1988                     if self.previousDepotPaths == []:
1989                         self.previousDepotPaths = depotPaths
1990                     else:
1991                         paths = []
1992                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1993                             prev_list = prev.split("/")
1994                             cur_list = cur.split("/")
1995                             for i in range(0, min(len(cur_list), len(prev_list))):
1996                                 if cur_list[i] <> prev_list[i]:
1997                                     i = i - 1
1998                                     break
1999
2000                             paths.append ("/".join(cur_list[:i + 1]))
2001
2002                         self.previousDepotPaths = paths
2003
2004             if p4Change > 0:
2005                 self.depotPaths = sorted(self.previousDepotPaths)
2006                 self.changeRange = "@%s,#head" % p4Change
2007                 if not self.detectBranches:
2008                     self.initialParent = parseRevision(self.branch)
2009                 if not self.silent and not self.detectBranches:
2010                     print "Performing incremental import into %s git branch" % self.branch
2011
2012         if not self.branch.startswith("refs/"):
2013             self.branch = "refs/heads/" + self.branch
2014
2015         if len(args) == 0 and self.depotPaths:
2016             if not self.silent:
2017                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2018         else:
2019             if self.depotPaths and self.depotPaths != args:
2020                 print ("previous import used depot path %s and now %s was specified. "
2021                        "This doesn't work!" % (' '.join (self.depotPaths),
2022                                                ' '.join (args)))
2023                 sys.exit(1)
2024
2025             self.depotPaths = sorted(args)
2026
2027         revision = ""
2028         self.users = {}
2029
2030         # Make sure no revision specifiers are used when --changesfile
2031         # is specified.
2032         bad_changesfile = False
2033         if len(self.changesFile) > 0:
2034             for p in self.depotPaths:
2035                 if p.find("@") >= 0 or p.find("#") >= 0:
2036                     bad_changesfile = True
2037                     break
2038         if bad_changesfile:
2039             die("Option --changesfile is incompatible with revision specifiers")
2040
2041         newPaths = []
2042         for p in self.depotPaths:
2043             if p.find("@") != -1:
2044                 atIdx = p.index("@")
2045                 self.changeRange = p[atIdx:]
2046                 if self.changeRange == "@all":
2047                     self.changeRange = ""
2048                 elif ',' not in self.changeRange:
2049                     revision = self.changeRange
2050                     self.changeRange = ""
2051                 p = p[:atIdx]
2052             elif p.find("#") != -1:
2053                 hashIdx = p.index("#")
2054                 revision = p[hashIdx:]
2055                 p = p[:hashIdx]
2056             elif self.previousDepotPaths == []:
2057                 # pay attention to changesfile, if given, else import
2058                 # the entire p4 tree at the head revision
2059                 if len(self.changesFile) == 0:
2060                     revision = "#head"
2061
2062             p = re.sub ("\.\.\.$", "", p)
2063             if not p.endswith("/"):
2064                 p += "/"
2065
2066             newPaths.append(p)
2067
2068         self.depotPaths = newPaths
2069
2070
2071         self.loadUserMapFromCache()
2072         self.labels = {}
2073         if self.detectLabels:
2074             self.getLabels();
2075
2076         if self.detectBranches:
2077             ## FIXME - what's a P4 projectName ?
2078             self.projectName = self.guessProjectName()
2079
2080             if self.hasOrigin:
2081                 self.getBranchMappingFromGitBranches()
2082             else:
2083                 self.getBranchMapping()
2084             if self.verbose:
2085                 print "p4-git branches: %s" % self.p4BranchesInGit
2086                 print "initial parents: %s" % self.initialParents
2087             for b in self.p4BranchesInGit:
2088                 if b != "master":
2089
2090                     ## FIXME
2091                     b = b[len(self.projectName):]
2092                 self.createdBranches.add(b)
2093
2094         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2095
2096         importProcess = subprocess.Popen(["git", "fast-import"],
2097                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2098                                          stderr=subprocess.PIPE);
2099         self.gitOutput = importProcess.stdout
2100         self.gitStream = importProcess.stdin
2101         self.gitError = importProcess.stderr
2102
2103         if revision:
2104             self.importHeadRevision(revision)
2105         else:
2106             changes = []
2107
2108             if len(self.changesFile) > 0:
2109                 output = open(self.changesFile).readlines()
2110                 changeSet = set()
2111                 for line in output:
2112                     changeSet.add(int(line))
2113
2114                 for change in changeSet:
2115                     changes.append(change)
2116
2117                 changes.sort()
2118             else:
2119                 # catch "git-p4 sync" with no new branches, in a repo that
2120                 # does not have any existing git-p4 branches
2121                 if len(args) == 0 and not self.p4BranchesInGit:
2122                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2123                 if self.verbose:
2124                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2125                                                               self.changeRange)
2126                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2127
2128                 if len(self.maxChanges) > 0:
2129                     changes = changes[:min(int(self.maxChanges), len(changes))]
2130
2131             if len(changes) == 0:
2132                 if not self.silent:
2133                     print "No changes to import!"
2134                 return True
2135
2136             if not self.silent and not self.detectBranches:
2137                 print "Import destination: %s" % self.branch
2138
2139             self.updatedBranches = set()
2140
2141             self.importChanges(changes)
2142
2143             if not self.silent:
2144                 print ""
2145                 if len(self.updatedBranches) > 0:
2146                     sys.stdout.write("Updated branches: ")
2147                     for b in self.updatedBranches:
2148                         sys.stdout.write("%s " % b)
2149                     sys.stdout.write("\n")
2150
2151         self.gitStream.close()
2152         if importProcess.wait() != 0:
2153             die("fast-import failed: %s" % self.gitError.read())
2154         self.gitOutput.close()
2155         self.gitError.close()
2156
2157         return True
2158
2159 class P4Rebase(Command):
2160     def __init__(self):
2161         Command.__init__(self)
2162         self.options = [ ]
2163         self.description = ("Fetches the latest revision from perforce and "
2164                             + "rebases the current work (branch) against it")
2165         self.verbose = False
2166
2167     def run(self, args):
2168         sync = P4Sync()
2169         sync.run([])
2170
2171         return self.rebase()
2172
2173     def rebase(self):
2174         if os.system("git update-index --refresh") != 0:
2175             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2176         if len(read_pipe("git diff-index HEAD --")) > 0:
2177             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2178
2179         [upstream, settings] = findUpstreamBranchPoint()
2180         if len(upstream) == 0:
2181             die("Cannot find upstream branchpoint for rebase")
2182
2183         # the branchpoint may be p4/foo~3, so strip off the parent
2184         upstream = re.sub("~[0-9]+$", "", upstream)
2185
2186         print "Rebasing the current branch onto %s" % upstream
2187         oldHead = read_pipe("git rev-parse HEAD").strip()
2188         system("git rebase %s" % upstream)
2189         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2190         return True
2191
2192 class P4Clone(P4Sync):
2193     def __init__(self):
2194         P4Sync.__init__(self)
2195         self.description = "Creates a new git repository and imports from Perforce into it"
2196         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2197         self.options += [
2198             optparse.make_option("--destination", dest="cloneDestination",
2199                                  action='store', default=None,
2200                                  help="where to leave result of the clone"),
2201             optparse.make_option("-/", dest="cloneExclude",
2202                                  action="append", type="string",
2203                                  help="exclude depot path"),
2204             optparse.make_option("--bare", dest="cloneBare",
2205                                  action="store_true", default=False),
2206         ]
2207         self.cloneDestination = None
2208         self.needsGit = False
2209         self.cloneBare = False
2210
2211     # This is required for the "append" cloneExclude action
2212     def ensure_value(self, attr, value):
2213         if not hasattr(self, attr) or getattr(self, attr) is None:
2214             setattr(self, attr, value)
2215         return getattr(self, attr)
2216
2217     def defaultDestination(self, args):
2218         ## TODO: use common prefix of args?
2219         depotPath = args[0]
2220         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2221         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2222         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2223         depotDir = re.sub(r"/$", "", depotDir)
2224         return os.path.split(depotDir)[1]
2225
2226     def run(self, args):
2227         if len(args) < 1:
2228             return False
2229
2230         if self.keepRepoPath and not self.cloneDestination:
2231             sys.stderr.write("Must specify destination for --keep-path\n")
2232             sys.exit(1)
2233
2234         depotPaths = args
2235
2236         if not self.cloneDestination and len(depotPaths) > 1:
2237             self.cloneDestination = depotPaths[-1]
2238             depotPaths = depotPaths[:-1]
2239
2240         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2241         for p in depotPaths:
2242             if not p.startswith("//"):
2243                 return False
2244
2245         if not self.cloneDestination:
2246             self.cloneDestination = self.defaultDestination(args)
2247
2248         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2249
2250         if not os.path.exists(self.cloneDestination):
2251             os.makedirs(self.cloneDestination)
2252         chdir(self.cloneDestination)
2253
2254         init_cmd = [ "git", "init" ]
2255         if self.cloneBare:
2256             init_cmd.append("--bare")
2257         subprocess.check_call(init_cmd)
2258
2259         if not P4Sync.run(self, depotPaths):
2260             return False
2261         if self.branch != "master":
2262             if self.importIntoRemotes:
2263                 masterbranch = "refs/remotes/p4/master"
2264             else:
2265                 masterbranch = "refs/heads/p4/master"
2266             if gitBranchExists(masterbranch):
2267                 system("git branch master %s" % masterbranch)
2268                 if not self.cloneBare:
2269                     system("git checkout -f")
2270             else:
2271                 print "Could not detect main branch. No checkout/master branch created."
2272
2273         return True
2274
2275 class P4Branches(Command):
2276     def __init__(self):
2277         Command.__init__(self)
2278         self.options = [ ]
2279         self.description = ("Shows the git branches that hold imports and their "
2280                             + "corresponding perforce depot paths")
2281         self.verbose = False
2282
2283     def run(self, args):
2284         if originP4BranchesExist():
2285             createOrUpdateBranchesFromOrigin()
2286
2287         cmdline = "git rev-parse --symbolic "
2288         cmdline += " --remotes"
2289
2290         for line in read_pipe_lines(cmdline):
2291             line = line.strip()
2292
2293             if not line.startswith('p4/') or line == "p4/HEAD":
2294                 continue
2295             branch = line
2296
2297             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2298             settings = extractSettingsGitLog(log)
2299
2300             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2301         return True
2302
2303 class HelpFormatter(optparse.IndentedHelpFormatter):
2304     def __init__(self):
2305         optparse.IndentedHelpFormatter.__init__(self)
2306
2307     def format_description(self, description):
2308         if description:
2309             return description + "\n"
2310         else:
2311             return ""
2312
2313 def printUsage(commands):
2314     print "usage: %s <command> [options]" % sys.argv[0]
2315     print ""
2316     print "valid commands: %s" % ", ".join(commands)
2317     print ""
2318     print "Try %s <command> --help for command specific help." % sys.argv[0]
2319     print ""
2320
2321 commands = {
2322     "debug" : P4Debug,
2323     "submit" : P4Submit,
2324     "commit" : P4Submit,
2325     "sync" : P4Sync,
2326     "rebase" : P4Rebase,
2327     "clone" : P4Clone,
2328     "rollback" : P4RollBack,
2329     "branches" : P4Branches
2330 }
2331
2332
2333 def main():
2334     if len(sys.argv[1:]) == 0:
2335         printUsage(commands.keys())
2336         sys.exit(2)
2337
2338     cmd = ""
2339     cmdName = sys.argv[1]
2340     try:
2341         klass = commands[cmdName]
2342         cmd = klass()
2343     except KeyError:
2344         print "unknown command %s" % cmdName
2345         print ""
2346         printUsage(commands.keys())
2347         sys.exit(2)
2348
2349     options = cmd.options
2350     cmd.gitdir = os.environ.get("GIT_DIR", None)
2351
2352     args = sys.argv[2:]
2353
2354     if len(options) > 0:
2355         if cmd.needsGit:
2356             options.append(optparse.make_option("--git-dir", dest="gitdir"))
2357
2358         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2359                                        options,
2360                                        description = cmd.description,
2361                                        formatter = HelpFormatter())
2362
2363         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2364     global verbose
2365     verbose = cmd.verbose
2366     if cmd.needsGit:
2367         if cmd.gitdir == None:
2368             cmd.gitdir = os.path.abspath(".git")
2369             if not isValidGitDir(cmd.gitdir):
2370                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2371                 if os.path.exists(cmd.gitdir):
2372                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2373                     if len(cdup) > 0:
2374                         chdir(cdup);
2375
2376         if not isValidGitDir(cmd.gitdir):
2377             if isValidGitDir(cmd.gitdir + "/.git"):
2378                 cmd.gitdir += "/.git"
2379             else:
2380                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2381
2382         os.environ["GIT_DIR"] = cmd.gitdir
2383
2384     if not cmd.run(args):
2385         parser.print_help()
2386         sys.exit(2)
2387
2388
2389 if __name__ == '__main__':
2390     main()