contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 def isP4Exec(kind):
 122     """Determine if a Perforce 'kind' should have execute permission
 123
 124     'p4 help filetypes' gives a list of the types.  If it starts with 'x',
 125     or x follows one of a few letters.  Otherwise, if there is an 'x' after
 126     a plus sign, it is also executable"""
 127     return (re.search(r"(^[cku]?x)|\+.*x", kind) != None)
 128
 129 def setP4ExecBit(file, mode):
 130     # Reopens an already open file and changes the execute bit to match
 131     # the execute bit setting in the passed in mode.
 132
 133     p4Type = "+x"
 134
 135     if not isModeExec(mode):
 136         p4Type = getP4OpenedType(file)
 137         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 138         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 139         if p4Type[-1] == "+":
 140             p4Type = p4Type[0:-1]
 141
 142     p4_system("reopen -t %s %s" % (p4Type, file))
 143
 144 def getP4OpenedType(file):
 145     # Returns the perforce file type for the given file.
 146
 147     result = p4_read_pipe("opened %s" % file)
 148     match = re.match(".*\((.+)\)\r?$", result)
 149     if match:
 150         return match.group(1)
 151     else:
 152         die("Could not determine file type for %s (result: '%s')" % (file, result))
 153
 154 def diffTreePattern():
 155     # This is a simple generator for the diff tree regex pattern. This could be
 156     # a class variable if this and parseDiffTreeEntry were a part of a class.
 157     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 158     while True:
 159         yield pattern
 160
 161 def parseDiffTreeEntry(entry):
 162     """Parses a single diff tree entry into its component elements.
 163
 164     See git-diff-tree(1) manpage for details about the format of the diff
 165     output. This method returns a dictionary with the following elements:
 166
 167     src_mode - The mode of the source file
 168     dst_mode - The mode of the destination file
 169     src_sha1 - The sha1 for the source file
 170     dst_sha1 - The sha1 fr the destination file
 171     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 172     status_score - The score for the status (applicable for 'C' and 'R'
 173                    statuses). This is None if there is no score.
 174     src - The path for the source file.
 175     dst - The path for the destination file. This is only present for
 176           copy or renames. If it is not present, this is None.
 177
 178     If the pattern is not matched, None is returned."""
 179
 180     match = diffTreePattern().next().match(entry)
 181     if match:
 182         return {
 183             'src_mode': match.group(1),
 184             'dst_mode': match.group(2),
 185             'src_sha1': match.group(3),
 186             'dst_sha1': match.group(4),
 187             'status': match.group(5),
 188             'status_score': match.group(6),
 189             'src': match.group(7),
 190             'dst': match.group(10)
 191         }
 192     return None
 193
 194 def isModeExec(mode):
 195     # Returns True if the given git mode represents an executable file,
 196     # otherwise False.
 197     return mode[-3:] == "755"
 198
 199 def isModeExecChanged(src_mode, dst_mode):
 200     return isModeExec(src_mode) != isModeExec(dst_mode)
 201
 202 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 203     cmd = p4_build_cmd("-G %s" % (cmd))
 204     if verbose:
 205         sys.stderr.write("Opening pipe: %s\n" % cmd)
 206
 207     # Use a temporary file to avoid deadlocks without
 208     # subprocess.communicate(), which would put another copy
 209     # of stdout into memory.
 210     stdin_file = None
 211     if stdin is not None:
 212         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 213         stdin_file.write(stdin)
 214         stdin_file.flush()
 215         stdin_file.seek(0)
 216
 217     p4 = subprocess.Popen(cmd, shell=True,
 218                           stdin=stdin_file,
 219                           stdout=subprocess.PIPE)
 220
 221     result = []
 222     try:
 223         while True:
 224             entry = marshal.load(p4.stdout)
 225             if cb is not None:
 226                 cb(entry)
 227             else:
 228                 result.append(entry)
 229     except EOFError:
 230         pass
 231     exitCode = p4.wait()
 232     if exitCode != 0:
 233         entry = {}
 234         entry["p4ExitCode"] = exitCode
 235         result.append(entry)
 236
 237     return result
 238
 239 def p4Cmd(cmd):
 240     list = p4CmdList(cmd)
 241     result = {}
 242     for entry in list:
 243         result.update(entry)
 244     return result;
 245
 246 def p4Where(depotPath):
 247     if not depotPath.endswith("/"):
 248         depotPath += "/"
 249     depotPath = depotPath + "..."
 250     outputList = p4CmdList("where %s" % depotPath)
 251     output = None
 252     for entry in outputList:
 253         if "depotFile" in entry:
 254             if entry["depotFile"] == depotPath:
 255                 output = entry
 256                 break
 257         elif "data" in entry:
 258             data = entry.get("data")
 259             space = data.find(" ")
 260             if data[:space] == depotPath:
 261                 output = entry
 262                 break
 263     if output == None:
 264         return ""
 265     if output["code"] == "error":
 266         return ""
 267     clientPath = ""
 268     if "path" in output:
 269         clientPath = output.get("path")
 270     elif "data" in output:
 271         data = output.get("data")
 272         lastSpace = data.rfind(" ")
 273         clientPath = data[lastSpace + 1:]
 274
 275     if clientPath.endswith("..."):
 276         clientPath = clientPath[:-3]
 277     return clientPath
 278
 279 def currentGitBranch():
 280     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 281
 282 def isValidGitDir(path):
 283     if (os.path.exists(path + "/HEAD")
 284         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 285         return True;
 286     return False
 287
 288 def parseRevision(ref):
 289     return read_pipe("git rev-parse %s" % ref).strip()
 290
 291 def extractLogMessageFromGitCommit(commit):
 292     logMessage = ""
 293
 294     ## fixme: title is first line of commit, not 1st paragraph.
 295     foundTitle = False
 296     for log in read_pipe_lines("git cat-file commit %s" % commit):
 297        if not foundTitle:
 298            if len(log) == 1:
 299                foundTitle = True
 300            continue
 301
 302        logMessage += log
 303     return logMessage
 304
 305 def extractSettingsGitLog(log):
 306     values = {}
 307     for line in log.split("\n"):
 308         line = line.strip()
 309         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 310         if not m:
 311             continue
 312
 313         assignments = m.group(1).split (':')
 314         for a in assignments:
 315             vals = a.split ('=')
 316             key = vals[0].strip()
 317             val = ('='.join (vals[1:])).strip()
 318             if val.endswith ('\"') and val.startswith('"'):
 319                 val = val[1:-1]
 320
 321             values[key] = val
 322
 323     paths = values.get("depot-paths")
 324     if not paths:
 325         paths = values.get("depot-path")
 326     if paths:
 327         values['depot-paths'] = paths.split(',')
 328     return values
 329
 330 def gitBranchExists(branch):
 331     proc = subprocess.Popen(["git", "rev-parse", branch],
 332                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 333     return proc.wait() == 0;
 334
 335 _gitConfig = {}
 336 def gitConfig(key, args = None): # set args to "--bool", for instance
 337     if not _gitConfig.has_key(key):
 338         argsFilter = ""
 339         if args != None:
 340             argsFilter = "%s " % args
 341         cmd = "git config %s%s" % (argsFilter, key)
 342         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 343     return _gitConfig[key]
 344
 345 def p4BranchesInGit(branchesAreInRemotes = True):
 346     branches = {}
 347
 348     cmdline = "git rev-parse --symbolic "
 349     if branchesAreInRemotes:
 350         cmdline += " --remotes"
 351     else:
 352         cmdline += " --branches"
 353
 354     for line in read_pipe_lines(cmdline):
 355         line = line.strip()
 356
 357         ## only import to p4/
 358         if not line.startswith('p4/') or line == "p4/HEAD":
 359             continue
 360         branch = line
 361
 362         # strip off p4
 363         branch = re.sub ("^p4/", "", line)
 364
 365         branches[branch] = parseRevision(line)
 366     return branches
 367
 368 def findUpstreamBranchPoint(head = "HEAD"):
 369     branches = p4BranchesInGit()
 370     # map from depot-path to branch name
 371     branchByDepotPath = {}
 372     for branch in branches.keys():
 373         tip = branches[branch]
 374         log = extractLogMessageFromGitCommit(tip)
 375         settings = extractSettingsGitLog(log)
 376         if settings.has_key("depot-paths"):
 377             paths = ",".join(settings["depot-paths"])
 378             branchByDepotPath[paths] = "remotes/p4/" + branch
 379
 380     settings = None
 381     parent = 0
 382     while parent < 65535:
 383         commit = head + "~%s" % parent
 384         log = extractLogMessageFromGitCommit(commit)
 385         settings = extractSettingsGitLog(log)
 386         if settings.has_key("depot-paths"):
 387             paths = ",".join(settings["depot-paths"])
 388             if branchByDepotPath.has_key(paths):
 389                 return [branchByDepotPath[paths], settings]
 390
 391         parent = parent + 1
 392
 393     return ["", settings]
 394
 395 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 396     if not silent:
 397         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 398                % localRefPrefix)
 399
 400     originPrefix = "origin/p4/"
 401
 402     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 403         line = line.strip()
 404         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 405             continue
 406
 407         headName = line[len(originPrefix):]
 408         remoteHead = localRefPrefix + headName
 409         originHead = line
 410
 411         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 412         if (not original.has_key('depot-paths')
 413             or not original.has_key('change')):
 414             continue
 415
 416         update = False
 417         if not gitBranchExists(remoteHead):
 418             if verbose:
 419                 print "creating %s" % remoteHead
 420             update = True
 421         else:
 422             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 423             if settings.has_key('change') > 0:
 424                 if settings['depot-paths'] == original['depot-paths']:
 425                     originP4Change = int(original['change'])
 426                     p4Change = int(settings['change'])
 427                     if originP4Change > p4Change:
 428                         print ("%s (%s) is newer than %s (%s). "
 429                                "Updating p4 branch from origin."
 430                                % (originHead, originP4Change,
 431                                   remoteHead, p4Change))
 432                         update = True
 433                 else:
 434                     print ("Ignoring: %s was imported from %s while "
 435                            "%s was imported from %s"
 436                            % (originHead, ','.join(original['depot-paths']),
 437                               remoteHead, ','.join(settings['depot-paths'])))
 438
 439         if update:
 440             system("git update-ref %s %s" % (remoteHead, originHead))
 441
 442 def originP4BranchesExist():
 443         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 444
 445 def p4ChangesForPaths(depotPaths, changeRange):
 446     assert depotPaths
 447     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 448                                                         for p in depotPaths]))
 449
 450     changes = {}
 451     for line in output:
 452         changeNum = int(line.split(" ")[1])
 453         changes[changeNum] = True
 454
 455     changelist = changes.keys()
 456     changelist.sort()
 457     return changelist
 458
 459 def p4PathStartsWith(path, prefix):
 460     # This method tries to remedy a potential mixed-case issue:
 461     #
 462     # If UserA adds  //depot/DirA/file1
 463     # and UserB adds //depot/dira/file2
 464     #
 465     # we may or may not have a problem. If you have core.ignorecase=true,
 466     # we treat DirA and dira as the same directory
 467     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 468     if ignorecase:
 469         return path.lower().startswith(prefix.lower())
 470     return path.startswith(prefix)
 471
 472 class Command:
 473     def __init__(self):
 474         self.usage = "usage: %prog [options]"
 475         self.needsGit = True
 476
 477 class P4UserMap:
 478     def __init__(self):
 479         self.userMapFromPerforceServer = False
 480
 481     def getUserCacheFilename(self):
 482         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 483         return home + "/.gitp4-usercache.txt"
 484
 485     def getUserMapFromPerforceServer(self):
 486         if self.userMapFromPerforceServer:
 487             return
 488         self.users = {}
 489         self.emails = {}
 490
 491         for output in p4CmdList("users"):
 492             if not output.has_key("User"):
 493                 continue
 494             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 495             self.emails[output["Email"]] = output["User"]
 496
 497
 498         s = ''
 499         for (key, val) in self.users.items():
 500             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 501
 502         open(self.getUserCacheFilename(), "wb").write(s)
 503         self.userMapFromPerforceServer = True
 504
 505     def loadUserMapFromCache(self):
 506         self.users = {}
 507         self.userMapFromPerforceServer = False
 508         try:
 509             cache = open(self.getUserCacheFilename(), "rb")
 510             lines = cache.readlines()
 511             cache.close()
 512             for line in lines:
 513                 entry = line.strip().split("\t")
 514                 self.users[entry[0]] = entry[1]
 515         except IOError:
 516             self.getUserMapFromPerforceServer()
 517
 518 class P4Debug(Command):
 519     def __init__(self):
 520         Command.__init__(self)
 521         self.options = [
 522             optparse.make_option("--verbose", dest="verbose", action="store_true",
 523                                  default=False),
 524             ]
 525         self.description = "A tool to debug the output of p4 -G."
 526         self.needsGit = False
 527         self.verbose = False
 528
 529     def run(self, args):
 530         j = 0
 531         for output in p4CmdList(" ".join(args)):
 532             print 'Element: %d' % j
 533             j += 1
 534             print output
 535         return True
 536
 537 class P4RollBack(Command):
 538     def __init__(self):
 539         Command.__init__(self)
 540         self.options = [
 541             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 542             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 543         ]
 544         self.description = "A tool to debug the multi-branch import. Don't use :)"
 545         self.verbose = False
 546         self.rollbackLocalBranches = False
 547
 548     def run(self, args):
 549         if len(args) != 1:
 550             return False
 551         maxChange = int(args[0])
 552
 553         if "p4ExitCode" in p4Cmd("changes -m 1"):
 554             die("Problems executing p4");
 555
 556         if self.rollbackLocalBranches:
 557             refPrefix = "refs/heads/"
 558             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 559         else:
 560             refPrefix = "refs/remotes/"
 561             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 562
 563         for line in lines:
 564             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 565                 line = line.strip()
 566                 ref = refPrefix + line
 567                 log = extractLogMessageFromGitCommit(ref)
 568                 settings = extractSettingsGitLog(log)
 569
 570                 depotPaths = settings['depot-paths']
 571                 change = settings['change']
 572
 573                 changed = False
 574
 575                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 576                                                            for p in depotPaths]))) == 0:
 577                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 578                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 579                     continue
 580
 581                 while change and int(change) > maxChange:
 582                     changed = True
 583                     if self.verbose:
 584                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 585                     system("git update-ref %s \"%s^\"" % (ref, ref))
 586                     log = extractLogMessageFromGitCommit(ref)
 587                     settings =  extractSettingsGitLog(log)
 588
 589
 590                     depotPaths = settings['depot-paths']
 591                     change = settings['change']
 592
 593                 if changed:
 594                     print "%s rewound to %s" % (ref, change)
 595
 596         return True
 597
 598 class P4Submit(Command, P4UserMap):
 599     def __init__(self):
 600         Command.__init__(self)
 601         P4UserMap.__init__(self)
 602         self.options = [
 603                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 604                 optparse.make_option("--origin", dest="origin"),
 605                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 606                 # preserve the user, requires relevant p4 permissions
 607                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 608         ]
 609         self.description = "Submit changes from git to the perforce depot."
 610         self.usage += " [name of git branch to submit into perforce depot]"
 611         self.interactive = True
 612         self.origin = ""
 613         self.detectRenames = False
 614         self.verbose = False
 615         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 616         self.isWindows = (platform.system() == "Windows")
 617         self.myP4UserId = None
 618
 619     def check(self):
 620         if len(p4CmdList("opened ...")) > 0:
 621             die("You have files opened with perforce! Close them before starting the sync.")
 622
 623     # replaces everything between 'Description:' and the next P4 submit template field with the
 624     # commit message
 625     def prepareLogMessage(self, template, message):
 626         result = ""
 627
 628         inDescriptionSection = False
 629
 630         for line in template.split("\n"):
 631             if line.startswith("#"):
 632                 result += line + "\n"
 633                 continue
 634
 635             if inDescriptionSection:
 636                 if line.startswith("Files:") or line.startswith("Jobs:"):
 637                     inDescriptionSection = False
 638                 else:
 639                     continue
 640             else:
 641                 if line.startswith("Description:"):
 642                     inDescriptionSection = True
 643                     line += "\n"
 644                     for messageLine in message.split("\n"):
 645                         line += "\t" + messageLine + "\n"
 646
 647             result += line + "\n"
 648
 649         return result
 650
 651     def p4UserForCommit(self,id):
 652         # Return the tuple (perforce user,git email) for a given git commit id
 653         self.getUserMapFromPerforceServer()
 654         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 655         gitEmail = gitEmail.strip()
 656         if not self.emails.has_key(gitEmail):
 657             return (None,gitEmail)
 658         else:
 659             return (self.emails[gitEmail],gitEmail)
 660
 661     def checkValidP4Users(self,commits):
 662         # check if any git authors cannot be mapped to p4 users
 663         for id in commits:
 664             (user,email) = self.p4UserForCommit(id)
 665             if not user:
 666                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 667                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 668                     print "%s" % msg
 669                 else:
 670                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 671
 672     def lastP4Changelist(self):
 673         # Get back the last changelist number submitted in this client spec. This
 674         # then gets used to patch up the username in the change. If the same
 675         # client spec is being used by multiple processes then this might go
 676         # wrong.
 677         results = p4CmdList("client -o")        # find the current client
 678         client = None
 679         for r in results:
 680             if r.has_key('Client'):
 681                 client = r['Client']
 682                 break
 683         if not client:
 684             die("could not get client spec")
 685         results = p4CmdList("changes -c %s -m 1" % client)
 686         for r in results:
 687             if r.has_key('change'):
 688                 return r['change']
 689         die("Could not get changelist number for last submit - cannot patch up user details")
 690
 691     def modifyChangelistUser(self, changelist, newUser):
 692         # fixup the user field of a changelist after it has been submitted.
 693         changes = p4CmdList("change -o %s" % changelist)
 694         if len(changes) != 1:
 695             die("Bad output from p4 change modifying %s to user %s" %
 696                 (changelist, newUser))
 697
 698         c = changes[0]
 699         if c['User'] == newUser: return   # nothing to do
 700         c['User'] = newUser
 701         input = marshal.dumps(c)
 702
 703         result = p4CmdList("change -f -i", stdin=input)
 704         for r in result:
 705             if r.has_key('code'):
 706                 if r['code'] == 'error':
 707                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 708             if r.has_key('data'):
 709                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 710                 return
 711         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 712
 713     def canChangeChangelists(self):
 714         # check to see if we have p4 admin or super-user permissions, either of
 715         # which are required to modify changelists.
 716         results = p4CmdList("protects %s" % self.depotPath)
 717         for r in results:
 718             if r.has_key('perm'):
 719                 if r['perm'] == 'admin':
 720                     return 1
 721                 if r['perm'] == 'super':
 722                     return 1
 723         return 0
 724
 725     def p4UserId(self):
 726         if self.myP4UserId:
 727             return self.myP4UserId
 728
 729         results = p4CmdList("user -o")
 730         for r in results:
 731             if r.has_key('User'):
 732                 self.myP4UserId = r['User']
 733                 return r['User']
 734         die("Could not find your p4 user id")
 735
 736     def p4UserIsMe(self, p4User):
 737         # return True if the given p4 user is actually me
 738         me = self.p4UserId()
 739         if not p4User or p4User != me:
 740             return False
 741         else:
 742             return True
 743
 744     def prepareSubmitTemplate(self):
 745         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 746         template = ""
 747         inFilesSection = False
 748         for line in p4_read_pipe_lines("change -o"):
 749             if line.endswith("\r\n"):
 750                 line = line[:-2] + "\n"
 751             if inFilesSection:
 752                 if line.startswith("\t"):
 753                     # path starts and ends with a tab
 754                     path = line[1:]
 755                     lastTab = path.rfind("\t")
 756                     if lastTab != -1:
 757                         path = path[:lastTab]
 758                         if not p4PathStartsWith(path, self.depotPath):
 759                             continue
 760                 else:
 761                     inFilesSection = False
 762             else:
 763                 if line.startswith("Files:"):
 764                     inFilesSection = True
 765
 766             template += line
 767
 768         return template
 769
 770     def applyCommit(self, id):
 771         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 772
 773         (p4User, gitEmail) = self.p4UserForCommit(id)
 774
 775         if not self.detectRenames:
 776             # If not explicitly set check the config variable
 777             self.detectRenames = gitConfig("git-p4.detectRenames")
 778
 779         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 780             diffOpts = ""
 781         elif self.detectRenames.lower() == "true":
 782             diffOpts = "-M"
 783         else:
 784             diffOpts = "-M%s" % self.detectRenames
 785
 786         detectCopies = gitConfig("git-p4.detectCopies")
 787         if detectCopies.lower() == "true":
 788             diffOpts += " -C"
 789         elif detectCopies != "" and detectCopies.lower() != "false":
 790             diffOpts += " -C%s" % detectCopies
 791
 792         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 793             diffOpts += " --find-copies-harder"
 794
 795         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 796         filesToAdd = set()
 797         filesToDelete = set()
 798         editedFiles = set()
 799         filesToChangeExecBit = {}
 800         for line in diff:
 801             diff = parseDiffTreeEntry(line)
 802             modifier = diff['status']
 803             path = diff['src']
 804             if modifier == "M":
 805                 p4_system("edit \"%s\"" % path)
 806                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 807                     filesToChangeExecBit[path] = diff['dst_mode']
 808                 editedFiles.add(path)
 809             elif modifier == "A":
 810                 filesToAdd.add(path)
 811                 filesToChangeExecBit[path] = diff['dst_mode']
 812                 if path in filesToDelete:
 813                     filesToDelete.remove(path)
 814             elif modifier == "D":
 815                 filesToDelete.add(path)
 816                 if path in filesToAdd:
 817                     filesToAdd.remove(path)
 818             elif modifier == "C":
 819                 src, dest = diff['src'], diff['dst']
 820                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 821                 if diff['src_sha1'] != diff['dst_sha1']:
 822                     p4_system("edit \"%s\"" % (dest))
 823                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 824                     p4_system("edit \"%s\"" % (dest))
 825                     filesToChangeExecBit[dest] = diff['dst_mode']
 826                 os.unlink(dest)
 827                 editedFiles.add(dest)
 828             elif modifier == "R":
 829                 src, dest = diff['src'], diff['dst']
 830                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 831                 if diff['src_sha1'] != diff['dst_sha1']:
 832                     p4_system("edit \"%s\"" % (dest))
 833                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 834                     p4_system("edit \"%s\"" % (dest))
 835                     filesToChangeExecBit[dest] = diff['dst_mode']
 836                 os.unlink(dest)
 837                 editedFiles.add(dest)
 838                 filesToDelete.add(src)
 839             else:
 840                 die("unknown modifier %s for %s" % (modifier, path))
 841
 842         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 843         patchcmd = diffcmd + " | git apply "
 844         tryPatchCmd = patchcmd + "--check -"
 845         applyPatchCmd = patchcmd + "--check --apply -"
 846
 847         if os.system(tryPatchCmd) != 0:
 848             print "Unfortunately applying the change failed!"
 849             print "What do you want to do?"
 850             response = "x"
 851             while response != "s" and response != "a" and response != "w":
 852                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 853                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 854             if response == "s":
 855                 print "Skipping! Good luck with the next patches..."
 856                 for f in editedFiles:
 857                     p4_system("revert \"%s\"" % f);
 858                 for f in filesToAdd:
 859                     system("rm %s" %f)
 860                 return
 861             elif response == "a":
 862                 os.system(applyPatchCmd)
 863                 if len(filesToAdd) > 0:
 864                     print "You may also want to call p4 add on the following files:"
 865                     print " ".join(filesToAdd)
 866                 if len(filesToDelete):
 867                     print "The following files should be scheduled for deletion with p4 delete:"
 868                     print " ".join(filesToDelete)
 869                 die("Please resolve and submit the conflict manually and "
 870                     + "continue afterwards with git-p4 submit --continue")
 871             elif response == "w":
 872                 system(diffcmd + " > patch.txt")
 873                 print "Patch saved to patch.txt in %s !" % self.clientPath
 874                 die("Please resolve and submit the conflict manually and "
 875                     "continue afterwards with git-p4 submit --continue")
 876
 877         system(applyPatchCmd)
 878
 879         for f in filesToAdd:
 880             p4_system("add \"%s\"" % f)
 881         for f in filesToDelete:
 882             p4_system("revert \"%s\"" % f)
 883             p4_system("delete \"%s\"" % f)
 884
 885         # Set/clear executable bits
 886         for f in filesToChangeExecBit.keys():
 887             mode = filesToChangeExecBit[f]
 888             setP4ExecBit(f, mode)
 889
 890         logMessage = extractLogMessageFromGitCommit(id)
 891         logMessage = logMessage.strip()
 892
 893         template = self.prepareSubmitTemplate()
 894
 895         if self.interactive:
 896             submitTemplate = self.prepareLogMessage(template, logMessage)
 897
 898             if self.preserveUser:
 899                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
 900
 901             if os.environ.has_key("P4DIFF"):
 902                 del(os.environ["P4DIFF"])
 903             diff = ""
 904             for editedFile in editedFiles:
 905                 diff += p4_read_pipe("diff -du %r" % editedFile)
 906
 907             newdiff = ""
 908             for newFile in filesToAdd:
 909                 newdiff += "==== new file ====\n"
 910                 newdiff += "--- /dev/null\n"
 911                 newdiff += "+++ %s\n" % newFile
 912                 f = open(newFile, "r")
 913                 for line in f.readlines():
 914                     newdiff += "+" + line
 915                 f.close()
 916
 917             if self.checkAuthorship and not self.p4UserIsMe(p4User):
 918                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
 919                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
 920                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
 921
 922             separatorLine = "######## everything below this line is just the diff #######\n"
 923
 924             [handle, fileName] = tempfile.mkstemp()
 925             tmpFile = os.fdopen(handle, "w+")
 926             if self.isWindows:
 927                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 928                 separatorLine = separatorLine.replace("\n", "\r\n")
 929                 newdiff = newdiff.replace("\n", "\r\n")
 930             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 931             tmpFile.close()
 932             mtime = os.stat(fileName).st_mtime
 933             if os.environ.has_key("P4EDITOR"):
 934                 editor = os.environ.get("P4EDITOR")
 935             else:
 936                 editor = read_pipe("git var GIT_EDITOR").strip()
 937             system(editor + " " + fileName)
 938
 939             if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 940                 checkModTime = False
 941             else:
 942                 checkModTime = True
 943
 944             response = "y"
 945             if checkModTime and (os.stat(fileName).st_mtime <= mtime):
 946                 response = "x"
 947                 while response != "y" and response != "n":
 948                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 949
 950             if response == "y":
 951                 tmpFile = open(fileName, "rb")
 952                 message = tmpFile.read()
 953                 tmpFile.close()
 954                 submitTemplate = message[:message.index(separatorLine)]
 955                 if self.isWindows:
 956                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 957                 p4_write_pipe("submit -i", submitTemplate)
 958
 959                 if self.preserveUser:
 960                     if p4User:
 961                         # Get last changelist number. Cannot easily get it from
 962                         # the submit command output as the output is unmarshalled.
 963                         changelist = self.lastP4Changelist()
 964                         self.modifyChangelistUser(changelist, p4User)
 965
 966             else:
 967                 for f in editedFiles:
 968                     p4_system("revert \"%s\"" % f);
 969                 for f in filesToAdd:
 970                     p4_system("revert \"%s\"" % f);
 971                     system("rm %s" %f)
 972
 973             os.remove(fileName)
 974         else:
 975             fileName = "submit.txt"
 976             file = open(fileName, "w+")
 977             file.write(self.prepareLogMessage(template, logMessage))
 978             file.close()
 979             print ("Perforce submit template written as %s. "
 980                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
 981                    % (fileName, fileName))
 982
 983     def run(self, args):
 984         if len(args) == 0:
 985             self.master = currentGitBranch()
 986             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
 987                 die("Detecting current git branch failed!")
 988         elif len(args) == 1:
 989             self.master = args[0]
 990         else:
 991             return False
 992
 993         allowSubmit = gitConfig("git-p4.allowSubmit")
 994         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
 995             die("%s is not in git-p4.allowSubmit" % self.master)
 996
 997         [upstream, settings] = findUpstreamBranchPoint()
 998         self.depotPath = settings['depot-paths'][0]
 999         if len(self.origin) == 0:
1000             self.origin = upstream
1001
1002         if self.preserveUser:
1003             if not self.canChangeChangelists():
1004                 die("Cannot preserve user names without p4 super-user or admin permissions")
1005
1006         if self.verbose:
1007             print "Origin branch is " + self.origin
1008
1009         if len(self.depotPath) == 0:
1010             print "Internal error: cannot locate perforce depot path from existing branches"
1011             sys.exit(128)
1012
1013         self.clientPath = p4Where(self.depotPath)
1014
1015         if len(self.clientPath) == 0:
1016             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1017             sys.exit(128)
1018
1019         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1020         self.oldWorkingDirectory = os.getcwd()
1021
1022         chdir(self.clientPath)
1023         print "Synchronizing p4 checkout..."
1024         p4_system("sync ...")
1025
1026         self.check()
1027
1028         commits = []
1029         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1030             commits.append(line.strip())
1031         commits.reverse()
1032
1033         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1034             self.checkAuthorship = False
1035         else:
1036             self.checkAuthorship = True
1037
1038         if self.preserveUser:
1039             self.checkValidP4Users(commits)
1040
1041         while len(commits) > 0:
1042             commit = commits[0]
1043             commits = commits[1:]
1044             self.applyCommit(commit)
1045             if not self.interactive:
1046                 break
1047
1048         if len(commits) == 0:
1049             print "All changes applied!"
1050             chdir(self.oldWorkingDirectory)
1051
1052             sync = P4Sync()
1053             sync.run([])
1054
1055             rebase = P4Rebase()
1056             rebase.rebase()
1057
1058         return True
1059
1060 class P4Sync(Command, P4UserMap):
1061     delete_actions = ( "delete", "move/delete", "purge" )
1062
1063     def __init__(self):
1064         Command.__init__(self)
1065         P4UserMap.__init__(self)
1066         self.options = [
1067                 optparse.make_option("--branch", dest="branch"),
1068                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1069                 optparse.make_option("--changesfile", dest="changesFile"),
1070                 optparse.make_option("--silent", dest="silent", action="store_true"),
1071                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1072                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1073                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1074                                      help="Import into refs/heads/ , not refs/remotes"),
1075                 optparse.make_option("--max-changes", dest="maxChanges"),
1076                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1077                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1078                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1079                                      help="Only sync files that are included in the Perforce Client Spec")
1080         ]
1081         self.description = """Imports from Perforce into a git repository.\n
1082     example:
1083     //depot/my/project/ -- to import the current head
1084     //depot/my/project/@all -- to import everything
1085     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1086
1087     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1088
1089         self.usage += " //depot/path[@revRange]"
1090         self.silent = False
1091         self.createdBranches = set()
1092         self.committedChanges = set()
1093         self.branch = ""
1094         self.detectBranches = False
1095         self.detectLabels = False
1096         self.changesFile = ""
1097         self.syncWithOrigin = True
1098         self.verbose = False
1099         self.importIntoRemotes = True
1100         self.maxChanges = ""
1101         self.isWindows = (platform.system() == "Windows")
1102         self.keepRepoPath = False
1103         self.depotPaths = None
1104         self.p4BranchesInGit = []
1105         self.cloneExclude = []
1106         self.useClientSpec = False
1107         self.clientSpecDirs = []
1108
1109         if gitConfig("git-p4.syncFromOrigin") == "false":
1110             self.syncWithOrigin = False
1111
1112     #
1113     # P4 wildcards are not allowed in filenames.  P4 complains
1114     # if you simply add them, but you can force it with "-f", in
1115     # which case it translates them into %xx encoding internally.
1116     # Search for and fix just these four characters.  Do % last so
1117     # that fixing it does not inadvertently create new %-escapes.
1118     #
1119     def wildcard_decode(self, path):
1120         # Cannot have * in a filename in windows; untested as to
1121         # what p4 would do in such a case.
1122         if not self.isWindows:
1123             path = path.replace("%2A", "*")
1124         path = path.replace("%23", "#") \
1125                    .replace("%40", "@") \
1126                    .replace("%25", "%")
1127         return path
1128
1129     def extractFilesFromCommit(self, commit):
1130         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1131                              for path in self.cloneExclude]
1132         files = []
1133         fnum = 0
1134         while commit.has_key("depotFile%s" % fnum):
1135             path =  commit["depotFile%s" % fnum]
1136
1137             if [p for p in self.cloneExclude
1138                 if p4PathStartsWith(path, p)]:
1139                 found = False
1140             else:
1141                 found = [p for p in self.depotPaths
1142                          if p4PathStartsWith(path, p)]
1143             if not found:
1144                 fnum = fnum + 1
1145                 continue
1146
1147             file = {}
1148             file["path"] = path
1149             file["rev"] = commit["rev%s" % fnum]
1150             file["action"] = commit["action%s" % fnum]
1151             file["type"] = commit["type%s" % fnum]
1152             files.append(file)
1153             fnum = fnum + 1
1154         return files
1155
1156     def stripRepoPath(self, path, prefixes):
1157         if self.useClientSpec:
1158
1159             # if using the client spec, we use the output directory
1160             # specified in the client.  For example, a view
1161             #   //depot/foo/branch/... //client/branch/foo/...
1162             # will end up putting all foo/branch files into
1163             #  branch/foo/
1164             for val in self.clientSpecDirs:
1165                 if path.startswith(val[0]):
1166                     # replace the depot path with the client path
1167                     path = path.replace(val[0], val[1][1])
1168                     # now strip out the client (//client/...)
1169                     path = re.sub("^(//[^/]+/)", '', path)
1170                     # the rest is all path
1171                     return path
1172
1173         if self.keepRepoPath:
1174             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1175
1176         for p in prefixes:
1177             if p4PathStartsWith(path, p):
1178                 path = path[len(p):]
1179
1180         return path
1181
1182     def splitFilesIntoBranches(self, commit):
1183         branches = {}
1184         fnum = 0
1185         while commit.has_key("depotFile%s" % fnum):
1186             path =  commit["depotFile%s" % fnum]
1187             found = [p for p in self.depotPaths
1188                      if p4PathStartsWith(path, p)]
1189             if not found:
1190                 fnum = fnum + 1
1191                 continue
1192
1193             file = {}
1194             file["path"] = path
1195             file["rev"] = commit["rev%s" % fnum]
1196             file["action"] = commit["action%s" % fnum]
1197             file["type"] = commit["type%s" % fnum]
1198             fnum = fnum + 1
1199
1200             relPath = self.stripRepoPath(path, self.depotPaths)
1201
1202             for branch in self.knownBranches.keys():
1203
1204                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1205                 if relPath.startswith(branch + "/"):
1206                     if branch not in branches:
1207                         branches[branch] = []
1208                     branches[branch].append(file)
1209                     break
1210
1211         return branches
1212
1213     # output one file from the P4 stream
1214     # - helper for streamP4Files
1215
1216     def streamOneP4File(self, file, contents):
1217         if file["type"] == "apple":
1218             print "\nfile %s is a strange apple file that forks. Ignoring" % \
1219                 file['depotFile']
1220             return
1221
1222         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1223         relPath = self.wildcard_decode(relPath)
1224         if verbose:
1225             sys.stderr.write("%s\n" % relPath)
1226
1227         mode = "644"
1228         if isP4Exec(file["type"]):
1229             mode = "755"
1230         elif file["type"] == "symlink":
1231             mode = "120000"
1232             # p4 print on a symlink contains "target\n", so strip it off
1233             data = ''.join(contents)
1234             contents = [data[:-1]]
1235
1236         if self.isWindows and file["type"].endswith("text"):
1237             mangled = []
1238             for data in contents:
1239                 data = data.replace("\r\n", "\n")
1240                 mangled.append(data)
1241             contents = mangled
1242
1243         if file['type'] in ('text+ko', 'unicode+ko', 'binary+ko'):
1244             contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$',r'$\1$', text), contents)
1245         elif file['type'] in ('text+k', 'ktext', 'kxtext', 'unicode+k', 'binary+k'):
1246             contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$',r'$\1$', text), contents)
1247
1248         self.gitStream.write("M %s inline %s\n" % (mode, relPath))
1249
1250         # total length...
1251         length = 0
1252         for d in contents:
1253             length = length + len(d)
1254
1255         self.gitStream.write("data %d\n" % length)
1256         for d in contents:
1257             self.gitStream.write(d)
1258         self.gitStream.write("\n")
1259
1260     def streamOneP4Deletion(self, file):
1261         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1262         if verbose:
1263             sys.stderr.write("delete %s\n" % relPath)
1264         self.gitStream.write("D %s\n" % relPath)
1265
1266     # handle another chunk of streaming data
1267     def streamP4FilesCb(self, marshalled):
1268
1269         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1270             # start of a new file - output the old one first
1271             self.streamOneP4File(self.stream_file, self.stream_contents)
1272             self.stream_file = {}
1273             self.stream_contents = []
1274             self.stream_have_file_info = False
1275
1276         # pick up the new file information... for the
1277         # 'data' field we need to append to our array
1278         for k in marshalled.keys():
1279             if k == 'data':
1280                 self.stream_contents.append(marshalled['data'])
1281             else:
1282                 self.stream_file[k] = marshalled[k]
1283
1284         self.stream_have_file_info = True
1285
1286     # Stream directly from "p4 files" into "git fast-import"
1287     def streamP4Files(self, files):
1288         filesForCommit = []
1289         filesToRead = []
1290         filesToDelete = []
1291
1292         for f in files:
1293             includeFile = True
1294             for val in self.clientSpecDirs:
1295                 if f['path'].startswith(val[0]):
1296                     if val[1][0] <= 0:
1297                         includeFile = False
1298                     break
1299
1300             if includeFile:
1301                 filesForCommit.append(f)
1302                 if f['action'] in self.delete_actions:
1303                     filesToDelete.append(f)
1304                 else:
1305                     filesToRead.append(f)
1306
1307         # deleted files...
1308         for f in filesToDelete:
1309             self.streamOneP4Deletion(f)
1310
1311         if len(filesToRead) > 0:
1312             self.stream_file = {}
1313             self.stream_contents = []
1314             self.stream_have_file_info = False
1315
1316             # curry self argument
1317             def streamP4FilesCbSelf(entry):
1318                 self.streamP4FilesCb(entry)
1319
1320             p4CmdList("-x - print",
1321                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1322                                                   for f in filesToRead]),
1323                 cb=streamP4FilesCbSelf)
1324
1325             # do the last chunk
1326             if self.stream_file.has_key('depotFile'):
1327                 self.streamOneP4File(self.stream_file, self.stream_contents)
1328
1329     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1330         epoch = details["time"]
1331         author = details["user"]
1332         self.branchPrefixes = branchPrefixes
1333
1334         if self.verbose:
1335             print "commit into %s" % branch
1336
1337         # start with reading files; if that fails, we should not
1338         # create a commit.
1339         new_files = []
1340         for f in files:
1341             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1342                 new_files.append (f)
1343             else:
1344                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1345
1346         self.gitStream.write("commit %s\n" % branch)
1347 #        gitStream.write("mark :%s\n" % details["change"])
1348         self.committedChanges.add(int(details["change"]))
1349         committer = ""
1350         if author not in self.users:
1351             self.getUserMapFromPerforceServer()
1352         if author in self.users:
1353             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1354         else:
1355             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1356
1357         self.gitStream.write("committer %s\n" % committer)
1358
1359         self.gitStream.write("data <<EOT\n")
1360         self.gitStream.write(details["desc"])
1361         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1362                              % (','.join (branchPrefixes), details["change"]))
1363         if len(details['options']) > 0:
1364             self.gitStream.write(": options = %s" % details['options'])
1365         self.gitStream.write("]\nEOT\n\n")
1366
1367         if len(parent) > 0:
1368             if self.verbose:
1369                 print "parent %s" % parent
1370             self.gitStream.write("from %s\n" % parent)
1371
1372         self.streamP4Files(new_files)
1373         self.gitStream.write("\n")
1374
1375         change = int(details["change"])
1376
1377         if self.labels.has_key(change):
1378             label = self.labels[change]
1379             labelDetails = label[0]
1380             labelRevisions = label[1]
1381             if self.verbose:
1382                 print "Change %s is labelled %s" % (change, labelDetails)
1383
1384             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1385                                                     for p in branchPrefixes]))
1386
1387             if len(files) == len(labelRevisions):
1388
1389                 cleanedFiles = {}
1390                 for info in files:
1391                     if info["action"] in self.delete_actions:
1392                         continue
1393                     cleanedFiles[info["depotFile"]] = info["rev"]
1394
1395                 if cleanedFiles == labelRevisions:
1396                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1397                     self.gitStream.write("from %s\n" % branch)
1398
1399                     owner = labelDetails["Owner"]
1400                     tagger = ""
1401                     if author in self.users:
1402                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1403                     else:
1404                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1405                     self.gitStream.write("tagger %s\n" % tagger)
1406                     self.gitStream.write("data <<EOT\n")
1407                     self.gitStream.write(labelDetails["Description"])
1408                     self.gitStream.write("EOT\n\n")
1409
1410                 else:
1411                     if not self.silent:
1412                         print ("Tag %s does not match with change %s: files do not match."
1413                                % (labelDetails["label"], change))
1414
1415             else:
1416                 if not self.silent:
1417                     print ("Tag %s does not match with change %s: file count is different."
1418                            % (labelDetails["label"], change))
1419
1420     def getLabels(self):
1421         self.labels = {}
1422
1423         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1424         if len(l) > 0 and not self.silent:
1425             print "Finding files belonging to labels in %s" % `self.depotPaths`
1426
1427         for output in l:
1428             label = output["label"]
1429             revisions = {}
1430             newestChange = 0
1431             if self.verbose:
1432                 print "Querying files for label %s" % label
1433             for file in p4CmdList("files "
1434                                   +  ' '.join (["%s...@%s" % (p, label)
1435                                                 for p in self.depotPaths])):
1436                 revisions[file["depotFile"]] = file["rev"]
1437                 change = int(file["change"])
1438                 if change > newestChange:
1439                     newestChange = change
1440
1441             self.labels[newestChange] = [output, revisions]
1442
1443         if self.verbose:
1444             print "Label changes: %s" % self.labels.keys()
1445
1446     def guessProjectName(self):
1447         for p in self.depotPaths:
1448             if p.endswith("/"):
1449                 p = p[:-1]
1450             p = p[p.strip().rfind("/") + 1:]
1451             if not p.endswith("/"):
1452                p += "/"
1453             return p
1454
1455     def getBranchMapping(self):
1456         lostAndFoundBranches = set()
1457
1458         for info in p4CmdList("branches"):
1459             details = p4Cmd("branch -o %s" % info["branch"])
1460             viewIdx = 0
1461             while details.has_key("View%s" % viewIdx):
1462                 paths = details["View%s" % viewIdx].split(" ")
1463                 viewIdx = viewIdx + 1
1464                 # require standard //depot/foo/... //depot/bar/... mapping
1465                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1466                     continue
1467                 source = paths[0]
1468                 destination = paths[1]
1469                 ## HACK
1470                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1471                     source = source[len(self.depotPaths[0]):-4]
1472                     destination = destination[len(self.depotPaths[0]):-4]
1473
1474                     if destination in self.knownBranches:
1475                         if not self.silent:
1476                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1477                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1478                         continue
1479
1480                     self.knownBranches[destination] = source
1481
1482                     lostAndFoundBranches.discard(destination)
1483
1484                     if source not in self.knownBranches:
1485                         lostAndFoundBranches.add(source)
1486
1487
1488         for branch in lostAndFoundBranches:
1489             self.knownBranches[branch] = branch
1490
1491     def getBranchMappingFromGitBranches(self):
1492         branches = p4BranchesInGit(self.importIntoRemotes)
1493         for branch in branches.keys():
1494             if branch == "master":
1495                 branch = "main"
1496             else:
1497                 branch = branch[len(self.projectName):]
1498             self.knownBranches[branch] = branch
1499
1500     def listExistingP4GitBranches(self):
1501         # branches holds mapping from name to commit
1502         branches = p4BranchesInGit(self.importIntoRemotes)
1503         self.p4BranchesInGit = branches.keys()
1504         for branch in branches.keys():
1505             self.initialParents[self.refPrefix + branch] = branches[branch]
1506
1507     def updateOptionDict(self, d):
1508         option_keys = {}
1509         if self.keepRepoPath:
1510             option_keys['keepRepoPath'] = 1
1511
1512         d["options"] = ' '.join(sorted(option_keys.keys()))
1513
1514     def readOptions(self, d):
1515         self.keepRepoPath = (d.has_key('options')
1516                              and ('keepRepoPath' in d['options']))
1517
1518     def gitRefForBranch(self, branch):
1519         if branch == "main":
1520             return self.refPrefix + "master"
1521
1522         if len(branch) <= 0:
1523             return branch
1524
1525         return self.refPrefix + self.projectName + branch
1526
1527     def gitCommitByP4Change(self, ref, change):
1528         if self.verbose:
1529             print "looking in ref " + ref + " for change %s using bisect..." % change
1530
1531         earliestCommit = ""
1532         latestCommit = parseRevision(ref)
1533
1534         while True:
1535             if self.verbose:
1536                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1537             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1538             if len(next) == 0:
1539                 if self.verbose:
1540                     print "argh"
1541                 return ""
1542             log = extractLogMessageFromGitCommit(next)
1543             settings = extractSettingsGitLog(log)
1544             currentChange = int(settings['change'])
1545             if self.verbose:
1546                 print "current change %s" % currentChange
1547
1548             if currentChange == change:
1549                 if self.verbose:
1550                     print "found %s" % next
1551                 return next
1552
1553             if currentChange < change:
1554                 earliestCommit = "^%s" % next
1555             else:
1556                 latestCommit = "%s" % next
1557
1558         return ""
1559
1560     def importNewBranch(self, branch, maxChange):
1561         # make fast-import flush all changes to disk and update the refs using the checkpoint
1562         # command so that we can try to find the branch parent in the git history
1563         self.gitStream.write("checkpoint\n\n");
1564         self.gitStream.flush();
1565         branchPrefix = self.depotPaths[0] + branch + "/"
1566         range = "@1,%s" % maxChange
1567         #print "prefix" + branchPrefix
1568         changes = p4ChangesForPaths([branchPrefix], range)
1569         if len(changes) <= 0:
1570             return False
1571         firstChange = changes[0]
1572         #print "first change in branch: %s" % firstChange
1573         sourceBranch = self.knownBranches[branch]
1574         sourceDepotPath = self.depotPaths[0] + sourceBranch
1575         sourceRef = self.gitRefForBranch(sourceBranch)
1576         #print "source " + sourceBranch
1577
1578         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1579         #print "branch parent: %s" % branchParentChange
1580         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1581         if len(gitParent) > 0:
1582             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1583             #print "parent git commit: %s" % gitParent
1584
1585         self.importChanges(changes)
1586         return True
1587
1588     def importChanges(self, changes):
1589         cnt = 1
1590         for change in changes:
1591             description = p4Cmd("describe %s" % change)
1592             self.updateOptionDict(description)
1593
1594             if not self.silent:
1595                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1596                 sys.stdout.flush()
1597             cnt = cnt + 1
1598
1599             try:
1600                 if self.detectBranches:
1601                     branches = self.splitFilesIntoBranches(description)
1602                     for branch in branches.keys():
1603                         ## HACK  --hwn
1604                         branchPrefix = self.depotPaths[0] + branch + "/"
1605
1606                         parent = ""
1607
1608                         filesForCommit = branches[branch]
1609
1610                         if self.verbose:
1611                             print "branch is %s" % branch
1612
1613                         self.updatedBranches.add(branch)
1614
1615                         if branch not in self.createdBranches:
1616                             self.createdBranches.add(branch)
1617                             parent = self.knownBranches[branch]
1618                             if parent == branch:
1619                                 parent = ""
1620                             else:
1621                                 fullBranch = self.projectName + branch
1622                                 if fullBranch not in self.p4BranchesInGit:
1623                                     if not self.silent:
1624                                         print("\n    Importing new branch %s" % fullBranch);
1625                                     if self.importNewBranch(branch, change - 1):
1626                                         parent = ""
1627                                         self.p4BranchesInGit.append(fullBranch)
1628                                     if not self.silent:
1629                                         print("\n    Resuming with change %s" % change);
1630
1631                                 if self.verbose:
1632                                     print "parent determined through known branches: %s" % parent
1633
1634                         branch = self.gitRefForBranch(branch)
1635                         parent = self.gitRefForBranch(parent)
1636
1637                         if self.verbose:
1638                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1639
1640                         if len(parent) == 0 and branch in self.initialParents:
1641                             parent = self.initialParents[branch]
1642                             del self.initialParents[branch]
1643
1644                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1645                 else:
1646                     files = self.extractFilesFromCommit(description)
1647                     self.commit(description, files, self.branch, self.depotPaths,
1648                                 self.initialParent)
1649                     self.initialParent = ""
1650             except IOError:
1651                 print self.gitError.read()
1652                 sys.exit(1)
1653
1654     def importHeadRevision(self, revision):
1655         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1656
1657         details = {}
1658         details["user"] = "git perforce import user"
1659         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1660                            % (' '.join(self.depotPaths), revision))
1661         details["change"] = revision
1662         newestRevision = 0
1663
1664         fileCnt = 0
1665         for info in p4CmdList("files "
1666                               +  ' '.join(["%s...%s"
1667                                            % (p, revision)
1668                                            for p in self.depotPaths])):
1669
1670             if 'code' in info and info['code'] == 'error':
1671                 sys.stderr.write("p4 returned an error: %s\n"
1672                                  % info['data'])
1673                 if info['data'].find("must refer to client") >= 0:
1674                     sys.stderr.write("This particular p4 error is misleading.\n")
1675                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1676                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1677                 sys.exit(1)
1678             if 'p4ExitCode' in info:
1679                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1680                 sys.exit(1)
1681
1682
1683             change = int(info["change"])
1684             if change > newestRevision:
1685                 newestRevision = change
1686
1687             if info["action"] in self.delete_actions:
1688                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1689                 #fileCnt = fileCnt + 1
1690                 continue
1691
1692             for prop in ["depotFile", "rev", "action", "type" ]:
1693                 details["%s%s" % (prop, fileCnt)] = info[prop]
1694
1695             fileCnt = fileCnt + 1
1696
1697         details["change"] = newestRevision
1698
1699         # Use time from top-most change so that all git-p4 clones of
1700         # the same p4 repo have the same commit SHA1s.
1701         res = p4CmdList("describe -s %d" % newestRevision)
1702         newestTime = None
1703         for r in res:
1704             if r.has_key('time'):
1705                 newestTime = int(r['time'])
1706         if newestTime is None:
1707             die("\"describe -s\" on newest change %d did not give a time")
1708         details["time"] = newestTime
1709
1710         self.updateOptionDict(details)
1711         try:
1712             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1713         except IOError:
1714             print "IO error with git fast-import. Is your git version recent enough?"
1715             print self.gitError.read()
1716
1717
1718     def getClientSpec(self):
1719         specList = p4CmdList( "client -o" )
1720         temp = {}
1721         for entry in specList:
1722             for k,v in entry.iteritems():
1723                 if k.startswith("View"):
1724
1725                     # p4 has these %%1 to %%9 arguments in specs to
1726                     # reorder paths; which we can't handle (yet :)
1727                     if re.match('%%\d', v) != None:
1728                         print "Sorry, can't handle %%n arguments in client specs"
1729                         sys.exit(1)
1730
1731                     if v.startswith('"'):
1732                         start = 1
1733                     else:
1734                         start = 0
1735                     index = v.find("...")
1736
1737                     # save the "client view"; i.e the RHS of the view
1738                     # line that tells the client where to put the
1739                     # files for this view.
1740                     cv = v[index+3:].strip() # +3 to remove previous '...'
1741
1742                     # if the client view doesn't end with a
1743                     # ... wildcard, then we're going to mess up the
1744                     # output directory, so fail gracefully.
1745                     if not cv.endswith('...'):
1746                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1747                         sys.exit(1)
1748                     cv=cv[:-3]
1749
1750                     # now save the view; +index means included, -index
1751                     # means it should be filtered out.
1752                     v = v[start:index]
1753                     if v.startswith("-"):
1754                         v = v[1:]
1755                         include = -len(v)
1756                     else:
1757                         include = len(v)
1758
1759                     temp[v] = (include, cv)
1760
1761         self.clientSpecDirs = temp.items()
1762         self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1763
1764     def run(self, args):
1765         self.depotPaths = []
1766         self.changeRange = ""
1767         self.initialParent = ""
1768         self.previousDepotPaths = []
1769
1770         # map from branch depot path to parent branch
1771         self.knownBranches = {}
1772         self.initialParents = {}
1773         self.hasOrigin = originP4BranchesExist()
1774         if not self.syncWithOrigin:
1775             self.hasOrigin = False
1776
1777         if self.importIntoRemotes:
1778             self.refPrefix = "refs/remotes/p4/"
1779         else:
1780             self.refPrefix = "refs/heads/p4/"
1781
1782         if self.syncWithOrigin and self.hasOrigin:
1783             if not self.silent:
1784                 print "Syncing with origin first by calling git fetch origin"
1785             system("git fetch origin")
1786
1787         if len(self.branch) == 0:
1788             self.branch = self.refPrefix + "master"
1789             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1790                 system("git update-ref %s refs/heads/p4" % self.branch)
1791                 system("git branch -D p4");
1792             # create it /after/ importing, when master exists
1793             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1794                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1795
1796         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1797             self.getClientSpec()
1798
1799         # TODO: should always look at previous commits,
1800         # merge with previous imports, if possible.
1801         if args == []:
1802             if self.hasOrigin:
1803                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1804             self.listExistingP4GitBranches()
1805
1806             if len(self.p4BranchesInGit) > 1:
1807                 if not self.silent:
1808                     print "Importing from/into multiple branches"
1809                 self.detectBranches = True
1810
1811             if self.verbose:
1812                 print "branches: %s" % self.p4BranchesInGit
1813
1814             p4Change = 0
1815             for branch in self.p4BranchesInGit:
1816                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1817
1818                 settings = extractSettingsGitLog(logMsg)
1819
1820                 self.readOptions(settings)
1821                 if (settings.has_key('depot-paths')
1822                     and settings.has_key ('change')):
1823                     change = int(settings['change']) + 1
1824                     p4Change = max(p4Change, change)
1825
1826                     depotPaths = sorted(settings['depot-paths'])
1827                     if self.previousDepotPaths == []:
1828                         self.previousDepotPaths = depotPaths
1829                     else:
1830                         paths = []
1831                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1832                             prev_list = prev.split("/")
1833                             cur_list = cur.split("/")
1834                             for i in range(0, min(len(cur_list), len(prev_list))):
1835                                 if cur_list[i] <> prev_list[i]:
1836                                     i = i - 1
1837                                     break
1838
1839                             paths.append ("/".join(cur_list[:i + 1]))
1840
1841                         self.previousDepotPaths = paths
1842
1843             if p4Change > 0:
1844                 self.depotPaths = sorted(self.previousDepotPaths)
1845                 self.changeRange = "@%s,#head" % p4Change
1846                 if not self.detectBranches:
1847                     self.initialParent = parseRevision(self.branch)
1848                 if not self.silent and not self.detectBranches:
1849                     print "Performing incremental import into %s git branch" % self.branch
1850
1851         if not self.branch.startswith("refs/"):
1852             self.branch = "refs/heads/" + self.branch
1853
1854         if len(args) == 0 and self.depotPaths:
1855             if not self.silent:
1856                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1857         else:
1858             if self.depotPaths and self.depotPaths != args:
1859                 print ("previous import used depot path %s and now %s was specified. "
1860                        "This doesn't work!" % (' '.join (self.depotPaths),
1861                                                ' '.join (args)))
1862                 sys.exit(1)
1863
1864             self.depotPaths = sorted(args)
1865
1866         revision = ""
1867         self.users = {}
1868
1869         newPaths = []
1870         for p in self.depotPaths:
1871             if p.find("@") != -1:
1872                 atIdx = p.index("@")
1873                 self.changeRange = p[atIdx:]
1874                 if self.changeRange == "@all":
1875                     self.changeRange = ""
1876                 elif ',' not in self.changeRange:
1877                     revision = self.changeRange
1878                     self.changeRange = ""
1879                 p = p[:atIdx]
1880             elif p.find("#") != -1:
1881                 hashIdx = p.index("#")
1882                 revision = p[hashIdx:]
1883                 p = p[:hashIdx]
1884             elif self.previousDepotPaths == []:
1885                 revision = "#head"
1886
1887             p = re.sub ("\.\.\.$", "", p)
1888             if not p.endswith("/"):
1889                 p += "/"
1890
1891             newPaths.append(p)
1892
1893         self.depotPaths = newPaths
1894
1895
1896         self.loadUserMapFromCache()
1897         self.labels = {}
1898         if self.detectLabels:
1899             self.getLabels();
1900
1901         if self.detectBranches:
1902             ## FIXME - what's a P4 projectName ?
1903             self.projectName = self.guessProjectName()
1904
1905             if self.hasOrigin:
1906                 self.getBranchMappingFromGitBranches()
1907             else:
1908                 self.getBranchMapping()
1909             if self.verbose:
1910                 print "p4-git branches: %s" % self.p4BranchesInGit
1911                 print "initial parents: %s" % self.initialParents
1912             for b in self.p4BranchesInGit:
1913                 if b != "master":
1914
1915                     ## FIXME
1916                     b = b[len(self.projectName):]
1917                 self.createdBranches.add(b)
1918
1919         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
1920
1921         importProcess = subprocess.Popen(["git", "fast-import"],
1922                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
1923                                          stderr=subprocess.PIPE);
1924         self.gitOutput = importProcess.stdout
1925         self.gitStream = importProcess.stdin
1926         self.gitError = importProcess.stderr
1927
1928         if revision:
1929             self.importHeadRevision(revision)
1930         else:
1931             changes = []
1932
1933             if len(self.changesFile) > 0:
1934                 output = open(self.changesFile).readlines()
1935                 changeSet = set()
1936                 for line in output:
1937                     changeSet.add(int(line))
1938
1939                 for change in changeSet:
1940                     changes.append(change)
1941
1942                 changes.sort()
1943             else:
1944                 # catch "git-p4 sync" with no new branches, in a repo that
1945                 # does not have any existing git-p4 branches
1946                 if len(args) == 0 and not self.p4BranchesInGit:
1947                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
1948                 if self.verbose:
1949                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
1950                                                               self.changeRange)
1951                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
1952
1953                 if len(self.maxChanges) > 0:
1954                     changes = changes[:min(int(self.maxChanges), len(changes))]
1955
1956             if len(changes) == 0:
1957                 if not self.silent:
1958                     print "No changes to import!"
1959                 return True
1960
1961             if not self.silent and not self.detectBranches:
1962                 print "Import destination: %s" % self.branch
1963
1964             self.updatedBranches = set()
1965
1966             self.importChanges(changes)
1967
1968             if not self.silent:
1969                 print ""
1970                 if len(self.updatedBranches) > 0:
1971                     sys.stdout.write("Updated branches: ")
1972                     for b in self.updatedBranches:
1973                         sys.stdout.write("%s " % b)
1974                     sys.stdout.write("\n")
1975
1976         self.gitStream.close()
1977         if importProcess.wait() != 0:
1978             die("fast-import failed: %s" % self.gitError.read())
1979         self.gitOutput.close()
1980         self.gitError.close()
1981
1982         return True
1983
1984 class P4Rebase(Command):
1985     def __init__(self):
1986         Command.__init__(self)
1987         self.options = [ ]
1988         self.description = ("Fetches the latest revision from perforce and "
1989                             + "rebases the current work (branch) against it")
1990         self.verbose = False
1991
1992     def run(self, args):
1993         sync = P4Sync()
1994         sync.run([])
1995
1996         return self.rebase()
1997
1998     def rebase(self):
1999         if os.system("git update-index --refresh") != 0:
2000             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2001         if len(read_pipe("git diff-index HEAD --")) > 0:
2002             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2003
2004         [upstream, settings] = findUpstreamBranchPoint()
2005         if len(upstream) == 0:
2006             die("Cannot find upstream branchpoint for rebase")
2007
2008         # the branchpoint may be p4/foo~3, so strip off the parent
2009         upstream = re.sub("~[0-9]+$", "", upstream)
2010
2011         print "Rebasing the current branch onto %s" % upstream
2012         oldHead = read_pipe("git rev-parse HEAD").strip()
2013         system("git rebase %s" % upstream)
2014         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2015         return True
2016
2017 class P4Clone(P4Sync):
2018     def __init__(self):
2019         P4Sync.__init__(self)
2020         self.description = "Creates a new git repository and imports from Perforce into it"
2021         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2022         self.options += [
2023             optparse.make_option("--destination", dest="cloneDestination",
2024                                  action='store', default=None,
2025                                  help="where to leave result of the clone"),
2026             optparse.make_option("-/", dest="cloneExclude",
2027                                  action="append", type="string",
2028                                  help="exclude depot path"),
2029             optparse.make_option("--bare", dest="cloneBare",
2030                                  action="store_true", default=False),
2031         ]
2032         self.cloneDestination = None
2033         self.needsGit = False
2034         self.cloneBare = False
2035
2036     # This is required for the "append" cloneExclude action
2037     def ensure_value(self, attr, value):
2038         if not hasattr(self, attr) or getattr(self, attr) is None:
2039             setattr(self, attr, value)
2040         return getattr(self, attr)
2041
2042     def defaultDestination(self, args):
2043         ## TODO: use common prefix of args?
2044         depotPath = args[0]
2045         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2046         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2047         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2048         depotDir = re.sub(r"/$", "", depotDir)
2049         return os.path.split(depotDir)[1]
2050
2051     def run(self, args):
2052         if len(args) < 1:
2053             return False
2054
2055         if self.keepRepoPath and not self.cloneDestination:
2056             sys.stderr.write("Must specify destination for --keep-path\n")
2057             sys.exit(1)
2058
2059         depotPaths = args
2060
2061         if not self.cloneDestination and len(depotPaths) > 1:
2062             self.cloneDestination = depotPaths[-1]
2063             depotPaths = depotPaths[:-1]
2064
2065         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2066         for p in depotPaths:
2067             if not p.startswith("//"):
2068                 return False
2069
2070         if not self.cloneDestination:
2071             self.cloneDestination = self.defaultDestination(args)
2072
2073         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2074
2075         if not os.path.exists(self.cloneDestination):
2076             os.makedirs(self.cloneDestination)
2077         chdir(self.cloneDestination)
2078
2079         init_cmd = [ "git", "init" ]
2080         if self.cloneBare:
2081             init_cmd.append("--bare")
2082         subprocess.check_call(init_cmd)
2083
2084         if not P4Sync.run(self, depotPaths):
2085             return False
2086         if self.branch != "master":
2087             if self.importIntoRemotes:
2088                 masterbranch = "refs/remotes/p4/master"
2089             else:
2090                 masterbranch = "refs/heads/p4/master"
2091             if gitBranchExists(masterbranch):
2092                 system("git branch master %s" % masterbranch)
2093                 if not self.cloneBare:
2094                     system("git checkout -f")
2095             else:
2096                 print "Could not detect main branch. No checkout/master branch created."
2097
2098         return True
2099
2100 class P4Branches(Command):
2101     def __init__(self):
2102         Command.__init__(self)
2103         self.options = [ ]
2104         self.description = ("Shows the git branches that hold imports and their "
2105                             + "corresponding perforce depot paths")
2106         self.verbose = False
2107
2108     def run(self, args):
2109         if originP4BranchesExist():
2110             createOrUpdateBranchesFromOrigin()
2111
2112         cmdline = "git rev-parse --symbolic "
2113         cmdline += " --remotes"
2114
2115         for line in read_pipe_lines(cmdline):
2116             line = line.strip()
2117
2118             if not line.startswith('p4/') or line == "p4/HEAD":
2119                 continue
2120             branch = line
2121
2122             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2123             settings = extractSettingsGitLog(log)
2124
2125             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2126         return True
2127
2128 class HelpFormatter(optparse.IndentedHelpFormatter):
2129     def __init__(self):
2130         optparse.IndentedHelpFormatter.__init__(self)
2131
2132     def format_description(self, description):
2133         if description:
2134             return description + "\n"
2135         else:
2136             return ""
2137
2138 def printUsage(commands):
2139     print "usage: %s <command> [options]" % sys.argv[0]
2140     print ""
2141     print "valid commands: %s" % ", ".join(commands)
2142     print ""
2143     print "Try %s <command> --help for command specific help." % sys.argv[0]
2144     print ""
2145
2146 commands = {
2147     "debug" : P4Debug,
2148     "submit" : P4Submit,
2149     "commit" : P4Submit,
2150     "sync" : P4Sync,
2151     "rebase" : P4Rebase,
2152     "clone" : P4Clone,
2153     "rollback" : P4RollBack,
2154     "branches" : P4Branches
2155 }
2156
2157
2158 def main():
2159     if len(sys.argv[1:]) == 0:
2160         printUsage(commands.keys())
2161         sys.exit(2)
2162
2163     cmd = ""
2164     cmdName = sys.argv[1]
2165     try:
2166         klass = commands[cmdName]
2167         cmd = klass()
2168     except KeyError:
2169         print "unknown command %s" % cmdName
2170         print ""
2171         printUsage(commands.keys())
2172         sys.exit(2)
2173
2174     options = cmd.options
2175     cmd.gitdir = os.environ.get("GIT_DIR", None)
2176
2177     args = sys.argv[2:]
2178
2179     if len(options) > 0:
2180         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2181
2182         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2183                                        options,
2184                                        description = cmd.description,
2185                                        formatter = HelpFormatter())
2186
2187         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2188     global verbose
2189     verbose = cmd.verbose
2190     if cmd.needsGit:
2191         if cmd.gitdir == None:
2192             cmd.gitdir = os.path.abspath(".git")
2193             if not isValidGitDir(cmd.gitdir):
2194                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2195                 if os.path.exists(cmd.gitdir):
2196                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2197                     if len(cdup) > 0:
2198                         chdir(cdup);
2199
2200         if not isValidGitDir(cmd.gitdir):
2201             if isValidGitDir(cmd.gitdir + "/.git"):
2202                 cmd.gitdir += "/.git"
2203             else:
2204                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2205
2206         os.environ["GIT_DIR"] = cmd.gitdir
2207
2208     if not cmd.run(args):
2209         parser.print_help()
2210
2211
2212 if __name__ == '__main__':
2213     main()