contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = "%s " % "p4"
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += "-u %s " % user
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += "-P %s " % password
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += "-p %s " % port
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += "-h %s " % host
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += "-c %s " % client
  46
  47     real_cmd += "%s" % (cmd)
  48     if verbose:
  49         print real_cmd
  50     return real_cmd
  51
  52 def chdir(dir):
  53     if os.name == 'nt':
  54         os.environ['PWD']=dir
  55     os.chdir(dir)
  56
  57 def die(msg):
  58     if verbose:
  59         raise Exception(msg)
  60     else:
  61         sys.stderr.write(msg + "\n")
  62         sys.exit(1)
  63
  64 def write_pipe(c, str):
  65     if verbose:
  66         sys.stderr.write('Writing pipe: %s\n' % c)
  67
  68     pipe = os.popen(c, 'w')
  69     val = pipe.write(str)
  70     if pipe.close():
  71         die('Command failed: %s' % c)
  72
  73     return val
  74
  75 def p4_write_pipe(c, str):
  76     real_cmd = p4_build_cmd(c)
  77     return write_pipe(real_cmd, str)
  78
  79 def read_pipe(c, ignore_error=False):
  80     if verbose:
  81         sys.stderr.write('Reading pipe: %s\n' % c)
  82
  83     pipe = os.popen(c, 'rb')
  84     val = pipe.read()
  85     if pipe.close() and not ignore_error:
  86         die('Command failed: %s' % c)
  87
  88     return val
  89
  90 def p4_read_pipe(c, ignore_error=False):
  91     real_cmd = p4_build_cmd(c)
  92     return read_pipe(real_cmd, ignore_error)
  93
  94 def read_pipe_lines(c):
  95     if verbose:
  96         sys.stderr.write('Reading pipe: %s\n' % c)
  97     ## todo: check return status
  98     pipe = os.popen(c, 'rb')
  99     val = pipe.readlines()
 100     if pipe.close():
 101         die('Command failed: %s' % c)
 102
 103     return val
 104
 105 def p4_read_pipe_lines(c):
 106     """Specifically invoke p4 on the command supplied. """
 107     real_cmd = p4_build_cmd(c)
 108     return read_pipe_lines(real_cmd)
 109
 110 def system(cmd):
 111     if verbose:
 112         sys.stderr.write("executing %s\n" % cmd)
 113     if os.system(cmd) != 0:
 114         die("command failed: %s" % cmd)
 115
 116 def p4_system(cmd):
 117     """Specifically invoke p4 as the system command. """
 118     real_cmd = p4_build_cmd(cmd)
 119     return system(real_cmd)
 120
 121 #
 122 # Canonicalize the p4 type and return a tuple of the
 123 # base type, plus any modifiers.  See "p4 help filetypes"
 124 # for a list and explanation.
 125 #
 126 def split_p4_type(p4type):
 127
 128     p4_filetypes_historical = {
 129         "ctempobj": "binary+Sw",
 130         "ctext": "text+C",
 131         "cxtext": "text+Cx",
 132         "ktext": "text+k",
 133         "kxtext": "text+kx",
 134         "ltext": "text+F",
 135         "tempobj": "binary+FSw",
 136         "ubinary": "binary+F",
 137         "uresource": "resource+F",
 138         "uxbinary": "binary+Fx",
 139         "xbinary": "binary+x",
 140         "xltext": "text+Fx",
 141         "xtempobj": "binary+Swx",
 142         "xtext": "text+x",
 143         "xunicode": "unicode+x",
 144         "xutf16": "utf16+x",
 145     }
 146     if p4type in p4_filetypes_historical:
 147         p4type = p4_filetypes_historical[p4type]
 148     mods = ""
 149     s = p4type.split("+")
 150     base = s[0]
 151     mods = ""
 152     if len(s) > 1:
 153         mods = s[1]
 154     return (base, mods)
 155
 156
 157 def setP4ExecBit(file, mode):
 158     # Reopens an already open file and changes the execute bit to match
 159     # the execute bit setting in the passed in mode.
 160
 161     p4Type = "+x"
 162
 163     if not isModeExec(mode):
 164         p4Type = getP4OpenedType(file)
 165         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 166         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 167         if p4Type[-1] == "+":
 168             p4Type = p4Type[0:-1]
 169
 170     p4_system("reopen -t %s %s" % (p4Type, file))
 171
 172 def getP4OpenedType(file):
 173     # Returns the perforce file type for the given file.
 174
 175     result = p4_read_pipe("opened %s" % file)
 176     match = re.match(".*\((.+)\)\r?$", result)
 177     if match:
 178         return match.group(1)
 179     else:
 180         die("Could not determine file type for %s (result: '%s')" % (file, result))
 181
 182 def diffTreePattern():
 183     # This is a simple generator for the diff tree regex pattern. This could be
 184     # a class variable if this and parseDiffTreeEntry were a part of a class.
 185     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 186     while True:
 187         yield pattern
 188
 189 def parseDiffTreeEntry(entry):
 190     """Parses a single diff tree entry into its component elements.
 191
 192     See git-diff-tree(1) manpage for details about the format of the diff
 193     output. This method returns a dictionary with the following elements:
 194
 195     src_mode - The mode of the source file
 196     dst_mode - The mode of the destination file
 197     src_sha1 - The sha1 for the source file
 198     dst_sha1 - The sha1 fr the destination file
 199     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 200     status_score - The score for the status (applicable for 'C' and 'R'
 201                    statuses). This is None if there is no score.
 202     src - The path for the source file.
 203     dst - The path for the destination file. This is only present for
 204           copy or renames. If it is not present, this is None.
 205
 206     If the pattern is not matched, None is returned."""
 207
 208     match = diffTreePattern().next().match(entry)
 209     if match:
 210         return {
 211             'src_mode': match.group(1),
 212             'dst_mode': match.group(2),
 213             'src_sha1': match.group(3),
 214             'dst_sha1': match.group(4),
 215             'status': match.group(5),
 216             'status_score': match.group(6),
 217             'src': match.group(7),
 218             'dst': match.group(10)
 219         }
 220     return None
 221
 222 def isModeExec(mode):
 223     # Returns True if the given git mode represents an executable file,
 224     # otherwise False.
 225     return mode[-3:] == "755"
 226
 227 def isModeExecChanged(src_mode, dst_mode):
 228     return isModeExec(src_mode) != isModeExec(dst_mode)
 229
 230 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 231     cmd = p4_build_cmd("-G %s" % (cmd))
 232     if verbose:
 233         sys.stderr.write("Opening pipe: %s\n" % cmd)
 234
 235     # Use a temporary file to avoid deadlocks without
 236     # subprocess.communicate(), which would put another copy
 237     # of stdout into memory.
 238     stdin_file = None
 239     if stdin is not None:
 240         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 241         stdin_file.write(stdin)
 242         stdin_file.flush()
 243         stdin_file.seek(0)
 244
 245     p4 = subprocess.Popen(cmd, shell=True,
 246                           stdin=stdin_file,
 247                           stdout=subprocess.PIPE)
 248
 249     result = []
 250     try:
 251         while True:
 252             entry = marshal.load(p4.stdout)
 253             if cb is not None:
 254                 cb(entry)
 255             else:
 256                 result.append(entry)
 257     except EOFError:
 258         pass
 259     exitCode = p4.wait()
 260     if exitCode != 0:
 261         entry = {}
 262         entry["p4ExitCode"] = exitCode
 263         result.append(entry)
 264
 265     return result
 266
 267 def p4Cmd(cmd):
 268     list = p4CmdList(cmd)
 269     result = {}
 270     for entry in list:
 271         result.update(entry)
 272     return result;
 273
 274 def p4Where(depotPath):
 275     if not depotPath.endswith("/"):
 276         depotPath += "/"
 277     depotPath = depotPath + "..."
 278     outputList = p4CmdList("where %s" % depotPath)
 279     output = None
 280     for entry in outputList:
 281         if "depotFile" in entry:
 282             if entry["depotFile"] == depotPath:
 283                 output = entry
 284                 break
 285         elif "data" in entry:
 286             data = entry.get("data")
 287             space = data.find(" ")
 288             if data[:space] == depotPath:
 289                 output = entry
 290                 break
 291     if output == None:
 292         return ""
 293     if output["code"] == "error":
 294         return ""
 295     clientPath = ""
 296     if "path" in output:
 297         clientPath = output.get("path")
 298     elif "data" in output:
 299         data = output.get("data")
 300         lastSpace = data.rfind(" ")
 301         clientPath = data[lastSpace + 1:]
 302
 303     if clientPath.endswith("..."):
 304         clientPath = clientPath[:-3]
 305     return clientPath
 306
 307 def currentGitBranch():
 308     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 309
 310 def isValidGitDir(path):
 311     if (os.path.exists(path + "/HEAD")
 312         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 313         return True;
 314     return False
 315
 316 def parseRevision(ref):
 317     return read_pipe("git rev-parse %s" % ref).strip()
 318
 319 def extractLogMessageFromGitCommit(commit):
 320     logMessage = ""
 321
 322     ## fixme: title is first line of commit, not 1st paragraph.
 323     foundTitle = False
 324     for log in read_pipe_lines("git cat-file commit %s" % commit):
 325        if not foundTitle:
 326            if len(log) == 1:
 327                foundTitle = True
 328            continue
 329
 330        logMessage += log
 331     return logMessage
 332
 333 def extractSettingsGitLog(log):
 334     values = {}
 335     for line in log.split("\n"):
 336         line = line.strip()
 337         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 338         if not m:
 339             continue
 340
 341         assignments = m.group(1).split (':')
 342         for a in assignments:
 343             vals = a.split ('=')
 344             key = vals[0].strip()
 345             val = ('='.join (vals[1:])).strip()
 346             if val.endswith ('\"') and val.startswith('"'):
 347                 val = val[1:-1]
 348
 349             values[key] = val
 350
 351     paths = values.get("depot-paths")
 352     if not paths:
 353         paths = values.get("depot-path")
 354     if paths:
 355         values['depot-paths'] = paths.split(',')
 356     return values
 357
 358 def gitBranchExists(branch):
 359     proc = subprocess.Popen(["git", "rev-parse", branch],
 360                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 361     return proc.wait() == 0;
 362
 363 _gitConfig = {}
 364 def gitConfig(key, args = None): # set args to "--bool", for instance
 365     if not _gitConfig.has_key(key):
 366         argsFilter = ""
 367         if args != None:
 368             argsFilter = "%s " % args
 369         cmd = "git config %s%s" % (argsFilter, key)
 370         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 371     return _gitConfig[key]
 372
 373 def gitConfigList(key):
 374     if not _gitConfig.has_key(key):
 375         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 376     return _gitConfig[key]
 377
 378 def p4BranchesInGit(branchesAreInRemotes = True):
 379     branches = {}
 380
 381     cmdline = "git rev-parse --symbolic "
 382     if branchesAreInRemotes:
 383         cmdline += " --remotes"
 384     else:
 385         cmdline += " --branches"
 386
 387     for line in read_pipe_lines(cmdline):
 388         line = line.strip()
 389
 390         ## only import to p4/
 391         if not line.startswith('p4/') or line == "p4/HEAD":
 392             continue
 393         branch = line
 394
 395         # strip off p4
 396         branch = re.sub ("^p4/", "", line)
 397
 398         branches[branch] = parseRevision(line)
 399     return branches
 400
 401 def findUpstreamBranchPoint(head = "HEAD"):
 402     branches = p4BranchesInGit()
 403     # map from depot-path to branch name
 404     branchByDepotPath = {}
 405     for branch in branches.keys():
 406         tip = branches[branch]
 407         log = extractLogMessageFromGitCommit(tip)
 408         settings = extractSettingsGitLog(log)
 409         if settings.has_key("depot-paths"):
 410             paths = ",".join(settings["depot-paths"])
 411             branchByDepotPath[paths] = "remotes/p4/" + branch
 412
 413     settings = None
 414     parent = 0
 415     while parent < 65535:
 416         commit = head + "~%s" % parent
 417         log = extractLogMessageFromGitCommit(commit)
 418         settings = extractSettingsGitLog(log)
 419         if settings.has_key("depot-paths"):
 420             paths = ",".join(settings["depot-paths"])
 421             if branchByDepotPath.has_key(paths):
 422                 return [branchByDepotPath[paths], settings]
 423
 424         parent = parent + 1
 425
 426     return ["", settings]
 427
 428 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 429     if not silent:
 430         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 431                % localRefPrefix)
 432
 433     originPrefix = "origin/p4/"
 434
 435     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 436         line = line.strip()
 437         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 438             continue
 439
 440         headName = line[len(originPrefix):]
 441         remoteHead = localRefPrefix + headName
 442         originHead = line
 443
 444         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 445         if (not original.has_key('depot-paths')
 446             or not original.has_key('change')):
 447             continue
 448
 449         update = False
 450         if not gitBranchExists(remoteHead):
 451             if verbose:
 452                 print "creating %s" % remoteHead
 453             update = True
 454         else:
 455             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 456             if settings.has_key('change') > 0:
 457                 if settings['depot-paths'] == original['depot-paths']:
 458                     originP4Change = int(original['change'])
 459                     p4Change = int(settings['change'])
 460                     if originP4Change > p4Change:
 461                         print ("%s (%s) is newer than %s (%s). "
 462                                "Updating p4 branch from origin."
 463                                % (originHead, originP4Change,
 464                                   remoteHead, p4Change))
 465                         update = True
 466                 else:
 467                     print ("Ignoring: %s was imported from %s while "
 468                            "%s was imported from %s"
 469                            % (originHead, ','.join(original['depot-paths']),
 470                               remoteHead, ','.join(settings['depot-paths'])))
 471
 472         if update:
 473             system("git update-ref %s %s" % (remoteHead, originHead))
 474
 475 def originP4BranchesExist():
 476         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 477
 478 def p4ChangesForPaths(depotPaths, changeRange):
 479     assert depotPaths
 480     output = p4_read_pipe_lines("changes " + ' '.join (["%s...%s" % (p, changeRange)
 481                                                         for p in depotPaths]))
 482
 483     changes = {}
 484     for line in output:
 485         changeNum = int(line.split(" ")[1])
 486         changes[changeNum] = True
 487
 488     changelist = changes.keys()
 489     changelist.sort()
 490     return changelist
 491
 492 def p4PathStartsWith(path, prefix):
 493     # This method tries to remedy a potential mixed-case issue:
 494     #
 495     # If UserA adds  //depot/DirA/file1
 496     # and UserB adds //depot/dira/file2
 497     #
 498     # we may or may not have a problem. If you have core.ignorecase=true,
 499     # we treat DirA and dira as the same directory
 500     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 501     if ignorecase:
 502         return path.lower().startswith(prefix.lower())
 503     return path.startswith(prefix)
 504
 505 class Command:
 506     def __init__(self):
 507         self.usage = "usage: %prog [options]"
 508         self.needsGit = True
 509
 510 class P4UserMap:
 511     def __init__(self):
 512         self.userMapFromPerforceServer = False
 513
 514     def getUserCacheFilename(self):
 515         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 516         return home + "/.gitp4-usercache.txt"
 517
 518     def getUserMapFromPerforceServer(self):
 519         if self.userMapFromPerforceServer:
 520             return
 521         self.users = {}
 522         self.emails = {}
 523
 524         for output in p4CmdList("users"):
 525             if not output.has_key("User"):
 526                 continue
 527             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 528             self.emails[output["Email"]] = output["User"]
 529
 530
 531         s = ''
 532         for (key, val) in self.users.items():
 533             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 534
 535         open(self.getUserCacheFilename(), "wb").write(s)
 536         self.userMapFromPerforceServer = True
 537
 538     def loadUserMapFromCache(self):
 539         self.users = {}
 540         self.userMapFromPerforceServer = False
 541         try:
 542             cache = open(self.getUserCacheFilename(), "rb")
 543             lines = cache.readlines()
 544             cache.close()
 545             for line in lines:
 546                 entry = line.strip().split("\t")
 547                 self.users[entry[0]] = entry[1]
 548         except IOError:
 549             self.getUserMapFromPerforceServer()
 550
 551 class P4Debug(Command):
 552     def __init__(self):
 553         Command.__init__(self)
 554         self.options = [
 555             optparse.make_option("--verbose", dest="verbose", action="store_true",
 556                                  default=False),
 557             ]
 558         self.description = "A tool to debug the output of p4 -G."
 559         self.needsGit = False
 560         self.verbose = False
 561
 562     def run(self, args):
 563         j = 0
 564         for output in p4CmdList(" ".join(args)):
 565             print 'Element: %d' % j
 566             j += 1
 567             print output
 568         return True
 569
 570 class P4RollBack(Command):
 571     def __init__(self):
 572         Command.__init__(self)
 573         self.options = [
 574             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 575             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 576         ]
 577         self.description = "A tool to debug the multi-branch import. Don't use :)"
 578         self.verbose = False
 579         self.rollbackLocalBranches = False
 580
 581     def run(self, args):
 582         if len(args) != 1:
 583             return False
 584         maxChange = int(args[0])
 585
 586         if "p4ExitCode" in p4Cmd("changes -m 1"):
 587             die("Problems executing p4");
 588
 589         if self.rollbackLocalBranches:
 590             refPrefix = "refs/heads/"
 591             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 592         else:
 593             refPrefix = "refs/remotes/"
 594             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 595
 596         for line in lines:
 597             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 598                 line = line.strip()
 599                 ref = refPrefix + line
 600                 log = extractLogMessageFromGitCommit(ref)
 601                 settings = extractSettingsGitLog(log)
 602
 603                 depotPaths = settings['depot-paths']
 604                 change = settings['change']
 605
 606                 changed = False
 607
 608                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 609                                                            for p in depotPaths]))) == 0:
 610                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 611                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 612                     continue
 613
 614                 while change and int(change) > maxChange:
 615                     changed = True
 616                     if self.verbose:
 617                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 618                     system("git update-ref %s \"%s^\"" % (ref, ref))
 619                     log = extractLogMessageFromGitCommit(ref)
 620                     settings =  extractSettingsGitLog(log)
 621
 622
 623                     depotPaths = settings['depot-paths']
 624                     change = settings['change']
 625
 626                 if changed:
 627                     print "%s rewound to %s" % (ref, change)
 628
 629         return True
 630
 631 class P4Submit(Command, P4UserMap):
 632     def __init__(self):
 633         Command.__init__(self)
 634         P4UserMap.__init__(self)
 635         self.options = [
 636                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 637                 optparse.make_option("--origin", dest="origin"),
 638                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 639                 # preserve the user, requires relevant p4 permissions
 640                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 641         ]
 642         self.description = "Submit changes from git to the perforce depot."
 643         self.usage += " [name of git branch to submit into perforce depot]"
 644         self.interactive = True
 645         self.origin = ""
 646         self.detectRenames = False
 647         self.verbose = False
 648         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 649         self.isWindows = (platform.system() == "Windows")
 650         self.myP4UserId = None
 651
 652     def check(self):
 653         if len(p4CmdList("opened ...")) > 0:
 654             die("You have files opened with perforce! Close them before starting the sync.")
 655
 656     # replaces everything between 'Description:' and the next P4 submit template field with the
 657     # commit message
 658     def prepareLogMessage(self, template, message):
 659         result = ""
 660
 661         inDescriptionSection = False
 662
 663         for line in template.split("\n"):
 664             if line.startswith("#"):
 665                 result += line + "\n"
 666                 continue
 667
 668             if inDescriptionSection:
 669                 if line.startswith("Files:") or line.startswith("Jobs:"):
 670                     inDescriptionSection = False
 671                 else:
 672                     continue
 673             else:
 674                 if line.startswith("Description:"):
 675                     inDescriptionSection = True
 676                     line += "\n"
 677                     for messageLine in message.split("\n"):
 678                         line += "\t" + messageLine + "\n"
 679
 680             result += line + "\n"
 681
 682         return result
 683
 684     def p4UserForCommit(self,id):
 685         # Return the tuple (perforce user,git email) for a given git commit id
 686         self.getUserMapFromPerforceServer()
 687         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 688         gitEmail = gitEmail.strip()
 689         if not self.emails.has_key(gitEmail):
 690             return (None,gitEmail)
 691         else:
 692             return (self.emails[gitEmail],gitEmail)
 693
 694     def checkValidP4Users(self,commits):
 695         # check if any git authors cannot be mapped to p4 users
 696         for id in commits:
 697             (user,email) = self.p4UserForCommit(id)
 698             if not user:
 699                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 700                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 701                     print "%s" % msg
 702                 else:
 703                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 704
 705     def lastP4Changelist(self):
 706         # Get back the last changelist number submitted in this client spec. This
 707         # then gets used to patch up the username in the change. If the same
 708         # client spec is being used by multiple processes then this might go
 709         # wrong.
 710         results = p4CmdList("client -o")        # find the current client
 711         client = None
 712         for r in results:
 713             if r.has_key('Client'):
 714                 client = r['Client']
 715                 break
 716         if not client:
 717             die("could not get client spec")
 718         results = p4CmdList("changes -c %s -m 1" % client)
 719         for r in results:
 720             if r.has_key('change'):
 721                 return r['change']
 722         die("Could not get changelist number for last submit - cannot patch up user details")
 723
 724     def modifyChangelistUser(self, changelist, newUser):
 725         # fixup the user field of a changelist after it has been submitted.
 726         changes = p4CmdList("change -o %s" % changelist)
 727         if len(changes) != 1:
 728             die("Bad output from p4 change modifying %s to user %s" %
 729                 (changelist, newUser))
 730
 731         c = changes[0]
 732         if c['User'] == newUser: return   # nothing to do
 733         c['User'] = newUser
 734         input = marshal.dumps(c)
 735
 736         result = p4CmdList("change -f -i", stdin=input)
 737         for r in result:
 738             if r.has_key('code'):
 739                 if r['code'] == 'error':
 740                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 741             if r.has_key('data'):
 742                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 743                 return
 744         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 745
 746     def canChangeChangelists(self):
 747         # check to see if we have p4 admin or super-user permissions, either of
 748         # which are required to modify changelists.
 749         results = p4CmdList("protects %s" % self.depotPath)
 750         for r in results:
 751             if r.has_key('perm'):
 752                 if r['perm'] == 'admin':
 753                     return 1
 754                 if r['perm'] == 'super':
 755                     return 1
 756         return 0
 757
 758     def p4UserId(self):
 759         if self.myP4UserId:
 760             return self.myP4UserId
 761
 762         results = p4CmdList("user -o")
 763         for r in results:
 764             if r.has_key('User'):
 765                 self.myP4UserId = r['User']
 766                 return r['User']
 767         die("Could not find your p4 user id")
 768
 769     def p4UserIsMe(self, p4User):
 770         # return True if the given p4 user is actually me
 771         me = self.p4UserId()
 772         if not p4User or p4User != me:
 773             return False
 774         else:
 775             return True
 776
 777     def prepareSubmitTemplate(self):
 778         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 779         template = ""
 780         inFilesSection = False
 781         for line in p4_read_pipe_lines("change -o"):
 782             if line.endswith("\r\n"):
 783                 line = line[:-2] + "\n"
 784             if inFilesSection:
 785                 if line.startswith("\t"):
 786                     # path starts and ends with a tab
 787                     path = line[1:]
 788                     lastTab = path.rfind("\t")
 789                     if lastTab != -1:
 790                         path = path[:lastTab]
 791                         if not p4PathStartsWith(path, self.depotPath):
 792                             continue
 793                 else:
 794                     inFilesSection = False
 795             else:
 796                 if line.startswith("Files:"):
 797                     inFilesSection = True
 798
 799             template += line
 800
 801         return template
 802
 803     def applyCommit(self, id):
 804         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 805
 806         (p4User, gitEmail) = self.p4UserForCommit(id)
 807
 808         if not self.detectRenames:
 809             # If not explicitly set check the config variable
 810             self.detectRenames = gitConfig("git-p4.detectRenames")
 811
 812         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 813             diffOpts = ""
 814         elif self.detectRenames.lower() == "true":
 815             diffOpts = "-M"
 816         else:
 817             diffOpts = "-M%s" % self.detectRenames
 818
 819         detectCopies = gitConfig("git-p4.detectCopies")
 820         if detectCopies.lower() == "true":
 821             diffOpts += " -C"
 822         elif detectCopies != "" and detectCopies.lower() != "false":
 823             diffOpts += " -C%s" % detectCopies
 824
 825         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 826             diffOpts += " --find-copies-harder"
 827
 828         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 829         filesToAdd = set()
 830         filesToDelete = set()
 831         editedFiles = set()
 832         filesToChangeExecBit = {}
 833         for line in diff:
 834             diff = parseDiffTreeEntry(line)
 835             modifier = diff['status']
 836             path = diff['src']
 837             if modifier == "M":
 838                 p4_system("edit \"%s\"" % path)
 839                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 840                     filesToChangeExecBit[path] = diff['dst_mode']
 841                 editedFiles.add(path)
 842             elif modifier == "A":
 843                 filesToAdd.add(path)
 844                 filesToChangeExecBit[path] = diff['dst_mode']
 845                 if path in filesToDelete:
 846                     filesToDelete.remove(path)
 847             elif modifier == "D":
 848                 filesToDelete.add(path)
 849                 if path in filesToAdd:
 850                     filesToAdd.remove(path)
 851             elif modifier == "C":
 852                 src, dest = diff['src'], diff['dst']
 853                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 854                 if diff['src_sha1'] != diff['dst_sha1']:
 855                     p4_system("edit \"%s\"" % (dest))
 856                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 857                     p4_system("edit \"%s\"" % (dest))
 858                     filesToChangeExecBit[dest] = diff['dst_mode']
 859                 os.unlink(dest)
 860                 editedFiles.add(dest)
 861             elif modifier == "R":
 862                 src, dest = diff['src'], diff['dst']
 863                 p4_system("integrate -Dt \"%s\" \"%s\"" % (src, dest))
 864                 if diff['src_sha1'] != diff['dst_sha1']:
 865                     p4_system("edit \"%s\"" % (dest))
 866                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 867                     p4_system("edit \"%s\"" % (dest))
 868                     filesToChangeExecBit[dest] = diff['dst_mode']
 869                 os.unlink(dest)
 870                 editedFiles.add(dest)
 871                 filesToDelete.add(src)
 872             else:
 873                 die("unknown modifier %s for %s" % (modifier, path))
 874
 875         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 876         patchcmd = diffcmd + " | git apply "
 877         tryPatchCmd = patchcmd + "--check -"
 878         applyPatchCmd = patchcmd + "--check --apply -"
 879
 880         if os.system(tryPatchCmd) != 0:
 881             print "Unfortunately applying the change failed!"
 882             print "What do you want to do?"
 883             response = "x"
 884             while response != "s" and response != "a" and response != "w":
 885                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 886                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 887             if response == "s":
 888                 print "Skipping! Good luck with the next patches..."
 889                 for f in editedFiles:
 890                     p4_system("revert \"%s\"" % f);
 891                 for f in filesToAdd:
 892                     system("rm %s" %f)
 893                 return
 894             elif response == "a":
 895                 os.system(applyPatchCmd)
 896                 if len(filesToAdd) > 0:
 897                     print "You may also want to call p4 add on the following files:"
 898                     print " ".join(filesToAdd)
 899                 if len(filesToDelete):
 900                     print "The following files should be scheduled for deletion with p4 delete:"
 901                     print " ".join(filesToDelete)
 902                 die("Please resolve and submit the conflict manually and "
 903                     + "continue afterwards with git-p4 submit --continue")
 904             elif response == "w":
 905                 system(diffcmd + " > patch.txt")
 906                 print "Patch saved to patch.txt in %s !" % self.clientPath
 907                 die("Please resolve and submit the conflict manually and "
 908                     "continue afterwards with git-p4 submit --continue")
 909
 910         system(applyPatchCmd)
 911
 912         for f in filesToAdd:
 913             p4_system("add \"%s\"" % f)
 914         for f in filesToDelete:
 915             p4_system("revert \"%s\"" % f)
 916             p4_system("delete \"%s\"" % f)
 917
 918         # Set/clear executable bits
 919         for f in filesToChangeExecBit.keys():
 920             mode = filesToChangeExecBit[f]
 921             setP4ExecBit(f, mode)
 922
 923         logMessage = extractLogMessageFromGitCommit(id)
 924         logMessage = logMessage.strip()
 925
 926         template = self.prepareSubmitTemplate()
 927
 928         if self.interactive:
 929             submitTemplate = self.prepareLogMessage(template, logMessage)
 930
 931             if self.preserveUser:
 932                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
 933
 934             if os.environ.has_key("P4DIFF"):
 935                 del(os.environ["P4DIFF"])
 936             diff = ""
 937             for editedFile in editedFiles:
 938                 diff += p4_read_pipe("diff -du %r" % editedFile)
 939
 940             newdiff = ""
 941             for newFile in filesToAdd:
 942                 newdiff += "==== new file ====\n"
 943                 newdiff += "--- /dev/null\n"
 944                 newdiff += "+++ %s\n" % newFile
 945                 f = open(newFile, "r")
 946                 for line in f.readlines():
 947                     newdiff += "+" + line
 948                 f.close()
 949
 950             if self.checkAuthorship and not self.p4UserIsMe(p4User):
 951                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
 952                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
 953                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
 954
 955             separatorLine = "######## everything below this line is just the diff #######\n"
 956
 957             [handle, fileName] = tempfile.mkstemp()
 958             tmpFile = os.fdopen(handle, "w+")
 959             if self.isWindows:
 960                 submitTemplate = submitTemplate.replace("\n", "\r\n")
 961                 separatorLine = separatorLine.replace("\n", "\r\n")
 962                 newdiff = newdiff.replace("\n", "\r\n")
 963             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
 964             tmpFile.close()
 965             mtime = os.stat(fileName).st_mtime
 966             if os.environ.has_key("P4EDITOR"):
 967                 editor = os.environ.get("P4EDITOR")
 968             else:
 969                 editor = read_pipe("git var GIT_EDITOR").strip()
 970             system(editor + " " + fileName)
 971
 972             if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 973                 checkModTime = False
 974             else:
 975                 checkModTime = True
 976
 977             response = "y"
 978             if checkModTime and (os.stat(fileName).st_mtime <= mtime):
 979                 response = "x"
 980                 while response != "y" and response != "n":
 981                     response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 982
 983             if response == "y":
 984                 tmpFile = open(fileName, "rb")
 985                 message = tmpFile.read()
 986                 tmpFile.close()
 987                 submitTemplate = message[:message.index(separatorLine)]
 988                 if self.isWindows:
 989                     submitTemplate = submitTemplate.replace("\r\n", "\n")
 990                 p4_write_pipe("submit -i", submitTemplate)
 991
 992                 if self.preserveUser:
 993                     if p4User:
 994                         # Get last changelist number. Cannot easily get it from
 995                         # the submit command output as the output is unmarshalled.
 996                         changelist = self.lastP4Changelist()
 997                         self.modifyChangelistUser(changelist, p4User)
 998
 999             else:
1000                 for f in editedFiles:
1001                     p4_system("revert \"%s\"" % f);
1002                 for f in filesToAdd:
1003                     p4_system("revert \"%s\"" % f);
1004                     system("rm %s" %f)
1005
1006             os.remove(fileName)
1007         else:
1008             fileName = "submit.txt"
1009             file = open(fileName, "w+")
1010             file.write(self.prepareLogMessage(template, logMessage))
1011             file.close()
1012             print ("Perforce submit template written as %s. "
1013                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1014                    % (fileName, fileName))
1015
1016     def run(self, args):
1017         if len(args) == 0:
1018             self.master = currentGitBranch()
1019             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1020                 die("Detecting current git branch failed!")
1021         elif len(args) == 1:
1022             self.master = args[0]
1023         else:
1024             return False
1025
1026         allowSubmit = gitConfig("git-p4.allowSubmit")
1027         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1028             die("%s is not in git-p4.allowSubmit" % self.master)
1029
1030         [upstream, settings] = findUpstreamBranchPoint()
1031         self.depotPath = settings['depot-paths'][0]
1032         if len(self.origin) == 0:
1033             self.origin = upstream
1034
1035         if self.preserveUser:
1036             if not self.canChangeChangelists():
1037                 die("Cannot preserve user names without p4 super-user or admin permissions")
1038
1039         if self.verbose:
1040             print "Origin branch is " + self.origin
1041
1042         if len(self.depotPath) == 0:
1043             print "Internal error: cannot locate perforce depot path from existing branches"
1044             sys.exit(128)
1045
1046         self.clientPath = p4Where(self.depotPath)
1047
1048         if len(self.clientPath) == 0:
1049             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1050             sys.exit(128)
1051
1052         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1053         self.oldWorkingDirectory = os.getcwd()
1054
1055         chdir(self.clientPath)
1056         print "Synchronizing p4 checkout..."
1057         p4_system("sync ...")
1058
1059         self.check()
1060
1061         commits = []
1062         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1063             commits.append(line.strip())
1064         commits.reverse()
1065
1066         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1067             self.checkAuthorship = False
1068         else:
1069             self.checkAuthorship = True
1070
1071         if self.preserveUser:
1072             self.checkValidP4Users(commits)
1073
1074         while len(commits) > 0:
1075             commit = commits[0]
1076             commits = commits[1:]
1077             self.applyCommit(commit)
1078             if not self.interactive:
1079                 break
1080
1081         if len(commits) == 0:
1082             print "All changes applied!"
1083             chdir(self.oldWorkingDirectory)
1084
1085             sync = P4Sync()
1086             sync.run([])
1087
1088             rebase = P4Rebase()
1089             rebase.rebase()
1090
1091         return True
1092
1093 class P4Sync(Command, P4UserMap):
1094     delete_actions = ( "delete", "move/delete", "purge" )
1095
1096     def __init__(self):
1097         Command.__init__(self)
1098         P4UserMap.__init__(self)
1099         self.options = [
1100                 optparse.make_option("--branch", dest="branch"),
1101                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1102                 optparse.make_option("--changesfile", dest="changesFile"),
1103                 optparse.make_option("--silent", dest="silent", action="store_true"),
1104                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1105                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1106                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1107                                      help="Import into refs/heads/ , not refs/remotes"),
1108                 optparse.make_option("--max-changes", dest="maxChanges"),
1109                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1110                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1111                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1112                                      help="Only sync files that are included in the Perforce Client Spec")
1113         ]
1114         self.description = """Imports from Perforce into a git repository.\n
1115     example:
1116     //depot/my/project/ -- to import the current head
1117     //depot/my/project/@all -- to import everything
1118     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1119
1120     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1121
1122         self.usage += " //depot/path[@revRange]"
1123         self.silent = False
1124         self.createdBranches = set()
1125         self.committedChanges = set()
1126         self.branch = ""
1127         self.detectBranches = False
1128         self.detectLabels = False
1129         self.changesFile = ""
1130         self.syncWithOrigin = True
1131         self.verbose = False
1132         self.importIntoRemotes = True
1133         self.maxChanges = ""
1134         self.isWindows = (platform.system() == "Windows")
1135         self.keepRepoPath = False
1136         self.depotPaths = None
1137         self.p4BranchesInGit = []
1138         self.cloneExclude = []
1139         self.useClientSpec = False
1140         self.clientSpecDirs = []
1141
1142         if gitConfig("git-p4.syncFromOrigin") == "false":
1143             self.syncWithOrigin = False
1144
1145     #
1146     # P4 wildcards are not allowed in filenames.  P4 complains
1147     # if you simply add them, but you can force it with "-f", in
1148     # which case it translates them into %xx encoding internally.
1149     # Search for and fix just these four characters.  Do % last so
1150     # that fixing it does not inadvertently create new %-escapes.
1151     #
1152     def wildcard_decode(self, path):
1153         # Cannot have * in a filename in windows; untested as to
1154         # what p4 would do in such a case.
1155         if not self.isWindows:
1156             path = path.replace("%2A", "*")
1157         path = path.replace("%23", "#") \
1158                    .replace("%40", "@") \
1159                    .replace("%25", "%")
1160         return path
1161
1162     def extractFilesFromCommit(self, commit):
1163         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1164                              for path in self.cloneExclude]
1165         files = []
1166         fnum = 0
1167         while commit.has_key("depotFile%s" % fnum):
1168             path =  commit["depotFile%s" % fnum]
1169
1170             if [p for p in self.cloneExclude
1171                 if p4PathStartsWith(path, p)]:
1172                 found = False
1173             else:
1174                 found = [p for p in self.depotPaths
1175                          if p4PathStartsWith(path, p)]
1176             if not found:
1177                 fnum = fnum + 1
1178                 continue
1179
1180             file = {}
1181             file["path"] = path
1182             file["rev"] = commit["rev%s" % fnum]
1183             file["action"] = commit["action%s" % fnum]
1184             file["type"] = commit["type%s" % fnum]
1185             files.append(file)
1186             fnum = fnum + 1
1187         return files
1188
1189     def stripRepoPath(self, path, prefixes):
1190         if self.useClientSpec:
1191
1192             # if using the client spec, we use the output directory
1193             # specified in the client.  For example, a view
1194             #   //depot/foo/branch/... //client/branch/foo/...
1195             # will end up putting all foo/branch files into
1196             #  branch/foo/
1197             for val in self.clientSpecDirs:
1198                 if path.startswith(val[0]):
1199                     # replace the depot path with the client path
1200                     path = path.replace(val[0], val[1][1])
1201                     # now strip out the client (//client/...)
1202                     path = re.sub("^(//[^/]+/)", '', path)
1203                     # the rest is all path
1204                     return path
1205
1206         if self.keepRepoPath:
1207             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1208
1209         for p in prefixes:
1210             if p4PathStartsWith(path, p):
1211                 path = path[len(p):]
1212
1213         return path
1214
1215     def splitFilesIntoBranches(self, commit):
1216         branches = {}
1217         fnum = 0
1218         while commit.has_key("depotFile%s" % fnum):
1219             path =  commit["depotFile%s" % fnum]
1220             found = [p for p in self.depotPaths
1221                      if p4PathStartsWith(path, p)]
1222             if not found:
1223                 fnum = fnum + 1
1224                 continue
1225
1226             file = {}
1227             file["path"] = path
1228             file["rev"] = commit["rev%s" % fnum]
1229             file["action"] = commit["action%s" % fnum]
1230             file["type"] = commit["type%s" % fnum]
1231             fnum = fnum + 1
1232
1233             relPath = self.stripRepoPath(path, self.depotPaths)
1234
1235             for branch in self.knownBranches.keys():
1236
1237                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1238                 if relPath.startswith(branch + "/"):
1239                     if branch not in branches:
1240                         branches[branch] = []
1241                     branches[branch].append(file)
1242                     break
1243
1244         return branches
1245
1246     # output one file from the P4 stream
1247     # - helper for streamP4Files
1248
1249     def streamOneP4File(self, file, contents):
1250         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1251         relPath = self.wildcard_decode(relPath)
1252         if verbose:
1253             sys.stderr.write("%s\n" % relPath)
1254
1255         (type_base, type_mods) = split_p4_type(file["type"])
1256
1257         git_mode = "100644"
1258         if "x" in type_mods:
1259             git_mode = "100755"
1260         if type_base == "symlink":
1261             git_mode = "120000"
1262             # p4 print on a symlink contains "target\n"; remove the newline
1263             data = ''.join(contents)
1264             contents = [data[:-1]]
1265
1266         if type_base == "utf16":
1267             # p4 delivers different text in the python output to -G
1268             # than it does when using "print -o", or normal p4 client
1269             # operations.  utf16 is converted to ascii or utf8, perhaps.
1270             # But ascii text saved as -t utf16 is completely mangled.
1271             # Invoke print -o to get the real contents.
1272             text = p4_read_pipe('print -q -o - "%s"' % file['depotFile'])
1273             contents = [ text ]
1274
1275         if type_base == "apple":
1276             # Apple filetype files will be streamed as a concatenation of
1277             # its appledouble header and the contents.  This is useless
1278             # on both macs and non-macs.  If using "print -q -o xx", it
1279             # will create "xx" with the data, and "%xx" with the header.
1280             # This is also not very useful.
1281             #
1282             # Ideally, someday, this script can learn how to generate
1283             # appledouble files directly and import those to git, but
1284             # non-mac machines can never find a use for apple filetype.
1285             print "\nIgnoring apple filetype file %s" % file['depotFile']
1286             return
1287
1288         # Perhaps windows wants unicode, utf16 newlines translated too;
1289         # but this is not doing it.
1290         if self.isWindows and type_base == "text":
1291             mangled = []
1292             for data in contents:
1293                 data = data.replace("\r\n", "\n")
1294                 mangled.append(data)
1295             contents = mangled
1296
1297         # Note that we do not try to de-mangle keywords on utf16 files,
1298         # even though in theory somebody may want that.
1299         if type_base in ("text", "unicode", "binary"):
1300             if "ko" in type_mods:
1301                 contents = map(lambda text: re.sub(r'(?i)\$(Id|Header):[^$]*\$', r'$\1$', text), contents)
1302             elif "k" in type_mods:
1303                 contents = map(lambda text: re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$\n]*\$', r'$\1$', text), contents)
1304
1305         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1306
1307         # total length...
1308         length = 0
1309         for d in contents:
1310             length = length + len(d)
1311
1312         self.gitStream.write("data %d\n" % length)
1313         for d in contents:
1314             self.gitStream.write(d)
1315         self.gitStream.write("\n")
1316
1317     def streamOneP4Deletion(self, file):
1318         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1319         if verbose:
1320             sys.stderr.write("delete %s\n" % relPath)
1321         self.gitStream.write("D %s\n" % relPath)
1322
1323     # handle another chunk of streaming data
1324     def streamP4FilesCb(self, marshalled):
1325
1326         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1327             # start of a new file - output the old one first
1328             self.streamOneP4File(self.stream_file, self.stream_contents)
1329             self.stream_file = {}
1330             self.stream_contents = []
1331             self.stream_have_file_info = False
1332
1333         # pick up the new file information... for the
1334         # 'data' field we need to append to our array
1335         for k in marshalled.keys():
1336             if k == 'data':
1337                 self.stream_contents.append(marshalled['data'])
1338             else:
1339                 self.stream_file[k] = marshalled[k]
1340
1341         self.stream_have_file_info = True
1342
1343     # Stream directly from "p4 files" into "git fast-import"
1344     def streamP4Files(self, files):
1345         filesForCommit = []
1346         filesToRead = []
1347         filesToDelete = []
1348
1349         for f in files:
1350             includeFile = True
1351             for val in self.clientSpecDirs:
1352                 if f['path'].startswith(val[0]):
1353                     if val[1][0] <= 0:
1354                         includeFile = False
1355                     break
1356
1357             if includeFile:
1358                 filesForCommit.append(f)
1359                 if f['action'] in self.delete_actions:
1360                     filesToDelete.append(f)
1361                 else:
1362                     filesToRead.append(f)
1363
1364         # deleted files...
1365         for f in filesToDelete:
1366             self.streamOneP4Deletion(f)
1367
1368         if len(filesToRead) > 0:
1369             self.stream_file = {}
1370             self.stream_contents = []
1371             self.stream_have_file_info = False
1372
1373             # curry self argument
1374             def streamP4FilesCbSelf(entry):
1375                 self.streamP4FilesCb(entry)
1376
1377             p4CmdList("-x - print",
1378                 '\n'.join(['%s#%s' % (f['path'], f['rev'])
1379                                                   for f in filesToRead]),
1380                 cb=streamP4FilesCbSelf)
1381
1382             # do the last chunk
1383             if self.stream_file.has_key('depotFile'):
1384                 self.streamOneP4File(self.stream_file, self.stream_contents)
1385
1386     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1387         epoch = details["time"]
1388         author = details["user"]
1389         self.branchPrefixes = branchPrefixes
1390
1391         if self.verbose:
1392             print "commit into %s" % branch
1393
1394         # start with reading files; if that fails, we should not
1395         # create a commit.
1396         new_files = []
1397         for f in files:
1398             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1399                 new_files.append (f)
1400             else:
1401                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1402
1403         self.gitStream.write("commit %s\n" % branch)
1404 #        gitStream.write("mark :%s\n" % details["change"])
1405         self.committedChanges.add(int(details["change"]))
1406         committer = ""
1407         if author not in self.users:
1408             self.getUserMapFromPerforceServer()
1409         if author in self.users:
1410             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1411         else:
1412             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1413
1414         self.gitStream.write("committer %s\n" % committer)
1415
1416         self.gitStream.write("data <<EOT\n")
1417         self.gitStream.write(details["desc"])
1418         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1419                              % (','.join (branchPrefixes), details["change"]))
1420         if len(details['options']) > 0:
1421             self.gitStream.write(": options = %s" % details['options'])
1422         self.gitStream.write("]\nEOT\n\n")
1423
1424         if len(parent) > 0:
1425             if self.verbose:
1426                 print "parent %s" % parent
1427             self.gitStream.write("from %s\n" % parent)
1428
1429         self.streamP4Files(new_files)
1430         self.gitStream.write("\n")
1431
1432         change = int(details["change"])
1433
1434         if self.labels.has_key(change):
1435             label = self.labels[change]
1436             labelDetails = label[0]
1437             labelRevisions = label[1]
1438             if self.verbose:
1439                 print "Change %s is labelled %s" % (change, labelDetails)
1440
1441             files = p4CmdList("files " + ' '.join (["%s...@%s" % (p, change)
1442                                                     for p in branchPrefixes]))
1443
1444             if len(files) == len(labelRevisions):
1445
1446                 cleanedFiles = {}
1447                 for info in files:
1448                     if info["action"] in self.delete_actions:
1449                         continue
1450                     cleanedFiles[info["depotFile"]] = info["rev"]
1451
1452                 if cleanedFiles == labelRevisions:
1453                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1454                     self.gitStream.write("from %s\n" % branch)
1455
1456                     owner = labelDetails["Owner"]
1457                     tagger = ""
1458                     if author in self.users:
1459                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1460                     else:
1461                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1462                     self.gitStream.write("tagger %s\n" % tagger)
1463                     self.gitStream.write("data <<EOT\n")
1464                     self.gitStream.write(labelDetails["Description"])
1465                     self.gitStream.write("EOT\n\n")
1466
1467                 else:
1468                     if not self.silent:
1469                         print ("Tag %s does not match with change %s: files do not match."
1470                                % (labelDetails["label"], change))
1471
1472             else:
1473                 if not self.silent:
1474                     print ("Tag %s does not match with change %s: file count is different."
1475                            % (labelDetails["label"], change))
1476
1477     def getLabels(self):
1478         self.labels = {}
1479
1480         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1481         if len(l) > 0 and not self.silent:
1482             print "Finding files belonging to labels in %s" % `self.depotPaths`
1483
1484         for output in l:
1485             label = output["label"]
1486             revisions = {}
1487             newestChange = 0
1488             if self.verbose:
1489                 print "Querying files for label %s" % label
1490             for file in p4CmdList("files "
1491                                   +  ' '.join (["%s...@%s" % (p, label)
1492                                                 for p in self.depotPaths])):
1493                 revisions[file["depotFile"]] = file["rev"]
1494                 change = int(file["change"])
1495                 if change > newestChange:
1496                     newestChange = change
1497
1498             self.labels[newestChange] = [output, revisions]
1499
1500         if self.verbose:
1501             print "Label changes: %s" % self.labels.keys()
1502
1503     def guessProjectName(self):
1504         for p in self.depotPaths:
1505             if p.endswith("/"):
1506                 p = p[:-1]
1507             p = p[p.strip().rfind("/") + 1:]
1508             if not p.endswith("/"):
1509                p += "/"
1510             return p
1511
1512     def getBranchMapping(self):
1513         lostAndFoundBranches = set()
1514
1515         user = gitConfig("git-p4.branchUser")
1516         if len(user) > 0:
1517             command = "branches -u %s" % user
1518         else:
1519             command = "branches"
1520
1521         for info in p4CmdList(command):
1522             details = p4Cmd("branch -o %s" % info["branch"])
1523             viewIdx = 0
1524             while details.has_key("View%s" % viewIdx):
1525                 paths = details["View%s" % viewIdx].split(" ")
1526                 viewIdx = viewIdx + 1
1527                 # require standard //depot/foo/... //depot/bar/... mapping
1528                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1529                     continue
1530                 source = paths[0]
1531                 destination = paths[1]
1532                 ## HACK
1533                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1534                     source = source[len(self.depotPaths[0]):-4]
1535                     destination = destination[len(self.depotPaths[0]):-4]
1536
1537                     if destination in self.knownBranches:
1538                         if not self.silent:
1539                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1540                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1541                         continue
1542
1543                     self.knownBranches[destination] = source
1544
1545                     lostAndFoundBranches.discard(destination)
1546
1547                     if source not in self.knownBranches:
1548                         lostAndFoundBranches.add(source)
1549
1550         # Perforce does not strictly require branches to be defined, so we also
1551         # check git config for a branch list.
1552         #
1553         # Example of branch definition in git config file:
1554         # [git-p4]
1555         #   branchList=main:branchA
1556         #   branchList=main:branchB
1557         #   branchList=branchA:branchC
1558         configBranches = gitConfigList("git-p4.branchList")
1559         for branch in configBranches:
1560             if branch:
1561                 (source, destination) = branch.split(":")
1562                 self.knownBranches[destination] = source
1563
1564                 lostAndFoundBranches.discard(destination)
1565
1566                 if source not in self.knownBranches:
1567                     lostAndFoundBranches.add(source)
1568
1569
1570         for branch in lostAndFoundBranches:
1571             self.knownBranches[branch] = branch
1572
1573     def getBranchMappingFromGitBranches(self):
1574         branches = p4BranchesInGit(self.importIntoRemotes)
1575         for branch in branches.keys():
1576             if branch == "master":
1577                 branch = "main"
1578             else:
1579                 branch = branch[len(self.projectName):]
1580             self.knownBranches[branch] = branch
1581
1582     def listExistingP4GitBranches(self):
1583         # branches holds mapping from name to commit
1584         branches = p4BranchesInGit(self.importIntoRemotes)
1585         self.p4BranchesInGit = branches.keys()
1586         for branch in branches.keys():
1587             self.initialParents[self.refPrefix + branch] = branches[branch]
1588
1589     def updateOptionDict(self, d):
1590         option_keys = {}
1591         if self.keepRepoPath:
1592             option_keys['keepRepoPath'] = 1
1593
1594         d["options"] = ' '.join(sorted(option_keys.keys()))
1595
1596     def readOptions(self, d):
1597         self.keepRepoPath = (d.has_key('options')
1598                              and ('keepRepoPath' in d['options']))
1599
1600     def gitRefForBranch(self, branch):
1601         if branch == "main":
1602             return self.refPrefix + "master"
1603
1604         if len(branch) <= 0:
1605             return branch
1606
1607         return self.refPrefix + self.projectName + branch
1608
1609     def gitCommitByP4Change(self, ref, change):
1610         if self.verbose:
1611             print "looking in ref " + ref + " for change %s using bisect..." % change
1612
1613         earliestCommit = ""
1614         latestCommit = parseRevision(ref)
1615
1616         while True:
1617             if self.verbose:
1618                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1619             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1620             if len(next) == 0:
1621                 if self.verbose:
1622                     print "argh"
1623                 return ""
1624             log = extractLogMessageFromGitCommit(next)
1625             settings = extractSettingsGitLog(log)
1626             currentChange = int(settings['change'])
1627             if self.verbose:
1628                 print "current change %s" % currentChange
1629
1630             if currentChange == change:
1631                 if self.verbose:
1632                     print "found %s" % next
1633                 return next
1634
1635             if currentChange < change:
1636                 earliestCommit = "^%s" % next
1637             else:
1638                 latestCommit = "%s" % next
1639
1640         return ""
1641
1642     def importNewBranch(self, branch, maxChange):
1643         # make fast-import flush all changes to disk and update the refs using the checkpoint
1644         # command so that we can try to find the branch parent in the git history
1645         self.gitStream.write("checkpoint\n\n");
1646         self.gitStream.flush();
1647         branchPrefix = self.depotPaths[0] + branch + "/"
1648         range = "@1,%s" % maxChange
1649         #print "prefix" + branchPrefix
1650         changes = p4ChangesForPaths([branchPrefix], range)
1651         if len(changes) <= 0:
1652             return False
1653         firstChange = changes[0]
1654         #print "first change in branch: %s" % firstChange
1655         sourceBranch = self.knownBranches[branch]
1656         sourceDepotPath = self.depotPaths[0] + sourceBranch
1657         sourceRef = self.gitRefForBranch(sourceBranch)
1658         #print "source " + sourceBranch
1659
1660         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1661         #print "branch parent: %s" % branchParentChange
1662         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1663         if len(gitParent) > 0:
1664             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1665             #print "parent git commit: %s" % gitParent
1666
1667         self.importChanges(changes)
1668         return True
1669
1670     def importChanges(self, changes):
1671         cnt = 1
1672         for change in changes:
1673             description = p4Cmd("describe %s" % change)
1674             self.updateOptionDict(description)
1675
1676             if not self.silent:
1677                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1678                 sys.stdout.flush()
1679             cnt = cnt + 1
1680
1681             try:
1682                 if self.detectBranches:
1683                     branches = self.splitFilesIntoBranches(description)
1684                     for branch in branches.keys():
1685                         ## HACK  --hwn
1686                         branchPrefix = self.depotPaths[0] + branch + "/"
1687
1688                         parent = ""
1689
1690                         filesForCommit = branches[branch]
1691
1692                         if self.verbose:
1693                             print "branch is %s" % branch
1694
1695                         self.updatedBranches.add(branch)
1696
1697                         if branch not in self.createdBranches:
1698                             self.createdBranches.add(branch)
1699                             parent = self.knownBranches[branch]
1700                             if parent == branch:
1701                                 parent = ""
1702                             else:
1703                                 fullBranch = self.projectName + branch
1704                                 if fullBranch not in self.p4BranchesInGit:
1705                                     if not self.silent:
1706                                         print("\n    Importing new branch %s" % fullBranch);
1707                                     if self.importNewBranch(branch, change - 1):
1708                                         parent = ""
1709                                         self.p4BranchesInGit.append(fullBranch)
1710                                     if not self.silent:
1711                                         print("\n    Resuming with change %s" % change);
1712
1713                                 if self.verbose:
1714                                     print "parent determined through known branches: %s" % parent
1715
1716                         branch = self.gitRefForBranch(branch)
1717                         parent = self.gitRefForBranch(parent)
1718
1719                         if self.verbose:
1720                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1721
1722                         if len(parent) == 0 and branch in self.initialParents:
1723                             parent = self.initialParents[branch]
1724                             del self.initialParents[branch]
1725
1726                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1727                 else:
1728                     files = self.extractFilesFromCommit(description)
1729                     self.commit(description, files, self.branch, self.depotPaths,
1730                                 self.initialParent)
1731                     self.initialParent = ""
1732             except IOError:
1733                 print self.gitError.read()
1734                 sys.exit(1)
1735
1736     def importHeadRevision(self, revision):
1737         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1738
1739         details = {}
1740         details["user"] = "git perforce import user"
1741         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1742                            % (' '.join(self.depotPaths), revision))
1743         details["change"] = revision
1744         newestRevision = 0
1745
1746         fileCnt = 0
1747         for info in p4CmdList("files "
1748                               +  ' '.join(["%s...%s"
1749                                            % (p, revision)
1750                                            for p in self.depotPaths])):
1751
1752             if 'code' in info and info['code'] == 'error':
1753                 sys.stderr.write("p4 returned an error: %s\n"
1754                                  % info['data'])
1755                 if info['data'].find("must refer to client") >= 0:
1756                     sys.stderr.write("This particular p4 error is misleading.\n")
1757                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1758                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1759                 sys.exit(1)
1760             if 'p4ExitCode' in info:
1761                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1762                 sys.exit(1)
1763
1764
1765             change = int(info["change"])
1766             if change > newestRevision:
1767                 newestRevision = change
1768
1769             if info["action"] in self.delete_actions:
1770                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1771                 #fileCnt = fileCnt + 1
1772                 continue
1773
1774             for prop in ["depotFile", "rev", "action", "type" ]:
1775                 details["%s%s" % (prop, fileCnt)] = info[prop]
1776
1777             fileCnt = fileCnt + 1
1778
1779         details["change"] = newestRevision
1780
1781         # Use time from top-most change so that all git-p4 clones of
1782         # the same p4 repo have the same commit SHA1s.
1783         res = p4CmdList("describe -s %d" % newestRevision)
1784         newestTime = None
1785         for r in res:
1786             if r.has_key('time'):
1787                 newestTime = int(r['time'])
1788         if newestTime is None:
1789             die("\"describe -s\" on newest change %d did not give a time")
1790         details["time"] = newestTime
1791
1792         self.updateOptionDict(details)
1793         try:
1794             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1795         except IOError:
1796             print "IO error with git fast-import. Is your git version recent enough?"
1797             print self.gitError.read()
1798
1799
1800     def getClientSpec(self):
1801         specList = p4CmdList( "client -o" )
1802         temp = {}
1803         for entry in specList:
1804             for k,v in entry.iteritems():
1805                 if k.startswith("View"):
1806
1807                     # p4 has these %%1 to %%9 arguments in specs to
1808                     # reorder paths; which we can't handle (yet :)
1809                     if re.match('%%\d', v) != None:
1810                         print "Sorry, can't handle %%n arguments in client specs"
1811                         sys.exit(1)
1812
1813                     if v.startswith('"'):
1814                         start = 1
1815                     else:
1816                         start = 0
1817                     index = v.find("...")
1818
1819                     # save the "client view"; i.e the RHS of the view
1820                     # line that tells the client where to put the
1821                     # files for this view.
1822                     cv = v[index+3:].strip() # +3 to remove previous '...'
1823
1824                     # if the client view doesn't end with a
1825                     # ... wildcard, then we're going to mess up the
1826                     # output directory, so fail gracefully.
1827                     if not cv.endswith('...'):
1828                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1829                         sys.exit(1)
1830                     cv=cv[:-3]
1831
1832                     # now save the view; +index means included, -index
1833                     # means it should be filtered out.
1834                     v = v[start:index]
1835                     if v.startswith("-"):
1836                         v = v[1:]
1837                         include = -len(v)
1838                     else:
1839                         include = len(v)
1840
1841                     temp[v] = (include, cv)
1842
1843         self.clientSpecDirs = temp.items()
1844         self.clientSpecDirs.sort( lambda x, y: abs( y[1][0] ) - abs( x[1][0] ) )
1845
1846     def run(self, args):
1847         self.depotPaths = []
1848         self.changeRange = ""
1849         self.initialParent = ""
1850         self.previousDepotPaths = []
1851
1852         # map from branch depot path to parent branch
1853         self.knownBranches = {}
1854         self.initialParents = {}
1855         self.hasOrigin = originP4BranchesExist()
1856         if not self.syncWithOrigin:
1857             self.hasOrigin = False
1858
1859         if self.importIntoRemotes:
1860             self.refPrefix = "refs/remotes/p4/"
1861         else:
1862             self.refPrefix = "refs/heads/p4/"
1863
1864         if self.syncWithOrigin and self.hasOrigin:
1865             if not self.silent:
1866                 print "Syncing with origin first by calling git fetch origin"
1867             system("git fetch origin")
1868
1869         if len(self.branch) == 0:
1870             self.branch = self.refPrefix + "master"
1871             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1872                 system("git update-ref %s refs/heads/p4" % self.branch)
1873                 system("git branch -D p4");
1874             # create it /after/ importing, when master exists
1875             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1876                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1877
1878         if self.useClientSpec or gitConfig("git-p4.useclientspec") == "true":
1879             self.getClientSpec()
1880
1881         # TODO: should always look at previous commits,
1882         # merge with previous imports, if possible.
1883         if args == []:
1884             if self.hasOrigin:
1885                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1886             self.listExistingP4GitBranches()
1887
1888             if len(self.p4BranchesInGit) > 1:
1889                 if not self.silent:
1890                     print "Importing from/into multiple branches"
1891                 self.detectBranches = True
1892
1893             if self.verbose:
1894                 print "branches: %s" % self.p4BranchesInGit
1895
1896             p4Change = 0
1897             for branch in self.p4BranchesInGit:
1898                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1899
1900                 settings = extractSettingsGitLog(logMsg)
1901
1902                 self.readOptions(settings)
1903                 if (settings.has_key('depot-paths')
1904                     and settings.has_key ('change')):
1905                     change = int(settings['change']) + 1
1906                     p4Change = max(p4Change, change)
1907
1908                     depotPaths = sorted(settings['depot-paths'])
1909                     if self.previousDepotPaths == []:
1910                         self.previousDepotPaths = depotPaths
1911                     else:
1912                         paths = []
1913                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
1914                             prev_list = prev.split("/")
1915                             cur_list = cur.split("/")
1916                             for i in range(0, min(len(cur_list), len(prev_list))):
1917                                 if cur_list[i] <> prev_list[i]:
1918                                     i = i - 1
1919                                     break
1920
1921                             paths.append ("/".join(cur_list[:i + 1]))
1922
1923                         self.previousDepotPaths = paths
1924
1925             if p4Change > 0:
1926                 self.depotPaths = sorted(self.previousDepotPaths)
1927                 self.changeRange = "@%s,#head" % p4Change
1928                 if not self.detectBranches:
1929                     self.initialParent = parseRevision(self.branch)
1930                 if not self.silent and not self.detectBranches:
1931                     print "Performing incremental import into %s git branch" % self.branch
1932
1933         if not self.branch.startswith("refs/"):
1934             self.branch = "refs/heads/" + self.branch
1935
1936         if len(args) == 0 and self.depotPaths:
1937             if not self.silent:
1938                 print "Depot paths: %s" % ' '.join(self.depotPaths)
1939         else:
1940             if self.depotPaths and self.depotPaths != args:
1941                 print ("previous import used depot path %s and now %s was specified. "
1942                        "This doesn't work!" % (' '.join (self.depotPaths),
1943                                                ' '.join (args)))
1944                 sys.exit(1)
1945
1946             self.depotPaths = sorted(args)
1947
1948         revision = ""
1949         self.users = {}
1950
1951         newPaths = []
1952         for p in self.depotPaths:
1953             if p.find("@") != -1:
1954                 atIdx = p.index("@")
1955                 self.changeRange = p[atIdx:]
1956                 if self.changeRange == "@all":
1957                     self.changeRange = ""
1958                 elif ',' not in self.changeRange:
1959                     revision = self.changeRange
1960                     self.changeRange = ""
1961                 p = p[:atIdx]
1962             elif p.find("#") != -1:
1963                 hashIdx = p.index("#")
1964                 revision = p[hashIdx:]
1965                 p = p[:hashIdx]
1966             elif self.previousDepotPaths == []:
1967                 revision = "#head"
1968
1969             p = re.sub ("\.\.\.$", "", p)
1970             if not p.endswith("/"):
1971                 p += "/"
1972
1973             newPaths.append(p)
1974
1975         self.depotPaths = newPaths
1976
1977
1978         self.loadUserMapFromCache()
1979         self.labels = {}
1980         if self.detectLabels:
1981             self.getLabels();
1982
1983         if self.detectBranches:
1984             ## FIXME - what's a P4 projectName ?
1985             self.projectName = self.guessProjectName()
1986
1987             if self.hasOrigin:
1988                 self.getBranchMappingFromGitBranches()
1989             else:
1990                 self.getBranchMapping()
1991             if self.verbose:
1992                 print "p4-git branches: %s" % self.p4BranchesInGit
1993                 print "initial parents: %s" % self.initialParents
1994             for b in self.p4BranchesInGit:
1995                 if b != "master":
1996
1997                     ## FIXME
1998                     b = b[len(self.projectName):]
1999                 self.createdBranches.add(b)
2000
2001         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2002
2003         importProcess = subprocess.Popen(["git", "fast-import"],
2004                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2005                                          stderr=subprocess.PIPE);
2006         self.gitOutput = importProcess.stdout
2007         self.gitStream = importProcess.stdin
2008         self.gitError = importProcess.stderr
2009
2010         if revision:
2011             self.importHeadRevision(revision)
2012         else:
2013             changes = []
2014
2015             if len(self.changesFile) > 0:
2016                 output = open(self.changesFile).readlines()
2017                 changeSet = set()
2018                 for line in output:
2019                     changeSet.add(int(line))
2020
2021                 for change in changeSet:
2022                     changes.append(change)
2023
2024                 changes.sort()
2025             else:
2026                 # catch "git-p4 sync" with no new branches, in a repo that
2027                 # does not have any existing git-p4 branches
2028                 if len(args) == 0 and not self.p4BranchesInGit:
2029                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2030                 if self.verbose:
2031                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2032                                                               self.changeRange)
2033                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2034
2035                 if len(self.maxChanges) > 0:
2036                     changes = changes[:min(int(self.maxChanges), len(changes))]
2037
2038             if len(changes) == 0:
2039                 if not self.silent:
2040                     print "No changes to import!"
2041                 return True
2042
2043             if not self.silent and not self.detectBranches:
2044                 print "Import destination: %s" % self.branch
2045
2046             self.updatedBranches = set()
2047
2048             self.importChanges(changes)
2049
2050             if not self.silent:
2051                 print ""
2052                 if len(self.updatedBranches) > 0:
2053                     sys.stdout.write("Updated branches: ")
2054                     for b in self.updatedBranches:
2055                         sys.stdout.write("%s " % b)
2056                     sys.stdout.write("\n")
2057
2058         self.gitStream.close()
2059         if importProcess.wait() != 0:
2060             die("fast-import failed: %s" % self.gitError.read())
2061         self.gitOutput.close()
2062         self.gitError.close()
2063
2064         return True
2065
2066 class P4Rebase(Command):
2067     def __init__(self):
2068         Command.__init__(self)
2069         self.options = [ ]
2070         self.description = ("Fetches the latest revision from perforce and "
2071                             + "rebases the current work (branch) against it")
2072         self.verbose = False
2073
2074     def run(self, args):
2075         sync = P4Sync()
2076         sync.run([])
2077
2078         return self.rebase()
2079
2080     def rebase(self):
2081         if os.system("git update-index --refresh") != 0:
2082             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2083         if len(read_pipe("git diff-index HEAD --")) > 0:
2084             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2085
2086         [upstream, settings] = findUpstreamBranchPoint()
2087         if len(upstream) == 0:
2088             die("Cannot find upstream branchpoint for rebase")
2089
2090         # the branchpoint may be p4/foo~3, so strip off the parent
2091         upstream = re.sub("~[0-9]+$", "", upstream)
2092
2093         print "Rebasing the current branch onto %s" % upstream
2094         oldHead = read_pipe("git rev-parse HEAD").strip()
2095         system("git rebase %s" % upstream)
2096         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2097         return True
2098
2099 class P4Clone(P4Sync):
2100     def __init__(self):
2101         P4Sync.__init__(self)
2102         self.description = "Creates a new git repository and imports from Perforce into it"
2103         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2104         self.options += [
2105             optparse.make_option("--destination", dest="cloneDestination",
2106                                  action='store', default=None,
2107                                  help="where to leave result of the clone"),
2108             optparse.make_option("-/", dest="cloneExclude",
2109                                  action="append", type="string",
2110                                  help="exclude depot path"),
2111             optparse.make_option("--bare", dest="cloneBare",
2112                                  action="store_true", default=False),
2113         ]
2114         self.cloneDestination = None
2115         self.needsGit = False
2116         self.cloneBare = False
2117
2118     # This is required for the "append" cloneExclude action
2119     def ensure_value(self, attr, value):
2120         if not hasattr(self, attr) or getattr(self, attr) is None:
2121             setattr(self, attr, value)
2122         return getattr(self, attr)
2123
2124     def defaultDestination(self, args):
2125         ## TODO: use common prefix of args?
2126         depotPath = args[0]
2127         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2128         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2129         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2130         depotDir = re.sub(r"/$", "", depotDir)
2131         return os.path.split(depotDir)[1]
2132
2133     def run(self, args):
2134         if len(args) < 1:
2135             return False
2136
2137         if self.keepRepoPath and not self.cloneDestination:
2138             sys.stderr.write("Must specify destination for --keep-path\n")
2139             sys.exit(1)
2140
2141         depotPaths = args
2142
2143         if not self.cloneDestination and len(depotPaths) > 1:
2144             self.cloneDestination = depotPaths[-1]
2145             depotPaths = depotPaths[:-1]
2146
2147         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2148         for p in depotPaths:
2149             if not p.startswith("//"):
2150                 return False
2151
2152         if not self.cloneDestination:
2153             self.cloneDestination = self.defaultDestination(args)
2154
2155         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2156
2157         if not os.path.exists(self.cloneDestination):
2158             os.makedirs(self.cloneDestination)
2159         chdir(self.cloneDestination)
2160
2161         init_cmd = [ "git", "init" ]
2162         if self.cloneBare:
2163             init_cmd.append("--bare")
2164         subprocess.check_call(init_cmd)
2165
2166         if not P4Sync.run(self, depotPaths):
2167             return False
2168         if self.branch != "master":
2169             if self.importIntoRemotes:
2170                 masterbranch = "refs/remotes/p4/master"
2171             else:
2172                 masterbranch = "refs/heads/p4/master"
2173             if gitBranchExists(masterbranch):
2174                 system("git branch master %s" % masterbranch)
2175                 if not self.cloneBare:
2176                     system("git checkout -f")
2177             else:
2178                 print "Could not detect main branch. No checkout/master branch created."
2179
2180         return True
2181
2182 class P4Branches(Command):
2183     def __init__(self):
2184         Command.__init__(self)
2185         self.options = [ ]
2186         self.description = ("Shows the git branches that hold imports and their "
2187                             + "corresponding perforce depot paths")
2188         self.verbose = False
2189
2190     def run(self, args):
2191         if originP4BranchesExist():
2192             createOrUpdateBranchesFromOrigin()
2193
2194         cmdline = "git rev-parse --symbolic "
2195         cmdline += " --remotes"
2196
2197         for line in read_pipe_lines(cmdline):
2198             line = line.strip()
2199
2200             if not line.startswith('p4/') or line == "p4/HEAD":
2201                 continue
2202             branch = line
2203
2204             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2205             settings = extractSettingsGitLog(log)
2206
2207             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2208         return True
2209
2210 class HelpFormatter(optparse.IndentedHelpFormatter):
2211     def __init__(self):
2212         optparse.IndentedHelpFormatter.__init__(self)
2213
2214     def format_description(self, description):
2215         if description:
2216             return description + "\n"
2217         else:
2218             return ""
2219
2220 def printUsage(commands):
2221     print "usage: %s <command> [options]" % sys.argv[0]
2222     print ""
2223     print "valid commands: %s" % ", ".join(commands)
2224     print ""
2225     print "Try %s <command> --help for command specific help." % sys.argv[0]
2226     print ""
2227
2228 commands = {
2229     "debug" : P4Debug,
2230     "submit" : P4Submit,
2231     "commit" : P4Submit,
2232     "sync" : P4Sync,
2233     "rebase" : P4Rebase,
2234     "clone" : P4Clone,
2235     "rollback" : P4RollBack,
2236     "branches" : P4Branches
2237 }
2238
2239
2240 def main():
2241     if len(sys.argv[1:]) == 0:
2242         printUsage(commands.keys())
2243         sys.exit(2)
2244
2245     cmd = ""
2246     cmdName = sys.argv[1]
2247     try:
2248         klass = commands[cmdName]
2249         cmd = klass()
2250     except KeyError:
2251         print "unknown command %s" % cmdName
2252         print ""
2253         printUsage(commands.keys())
2254         sys.exit(2)
2255
2256     options = cmd.options
2257     cmd.gitdir = os.environ.get("GIT_DIR", None)
2258
2259     args = sys.argv[2:]
2260
2261     if len(options) > 0:
2262         options.append(optparse.make_option("--git-dir", dest="gitdir"))
2263
2264         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2265                                        options,
2266                                        description = cmd.description,
2267                                        formatter = HelpFormatter())
2268
2269         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2270     global verbose
2271     verbose = cmd.verbose
2272     if cmd.needsGit:
2273         if cmd.gitdir == None:
2274             cmd.gitdir = os.path.abspath(".git")
2275             if not isValidGitDir(cmd.gitdir):
2276                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2277                 if os.path.exists(cmd.gitdir):
2278                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2279                     if len(cdup) > 0:
2280                         chdir(cdup);
2281
2282         if not isValidGitDir(cmd.gitdir):
2283             if isValidGitDir(cmd.gitdir + "/.git"):
2284                 cmd.gitdir += "/.git"
2285             else:
2286                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2287
2288         os.environ["GIT_DIR"] = cmd.gitdir
2289
2290     if not cmd.run(args):
2291         parser.print_help()
2292
2293
2294 if __name__ == '__main__':
2295     main()