contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <simon@lst.de>
   6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, subprocess, shelve
  12 import tempfile, getopt, os.path, time, platform
  13 import re
  14
  15 verbose = False
  16
  17
  18 def p4_build_cmd(cmd):
  19     """Build a suitable p4 command line.
  20
  21     This consolidates building and returning a p4 command line into one
  22     location. It means that hooking into the environment, or other configuration
  23     can be done more easily.
  24     """
  25     real_cmd = ["p4"]
  26
  27     user = gitConfig("git-p4.user")
  28     if len(user) > 0:
  29         real_cmd += ["-u",user]
  30
  31     password = gitConfig("git-p4.password")
  32     if len(password) > 0:
  33         real_cmd += ["-P", password]
  34
  35     port = gitConfig("git-p4.port")
  36     if len(port) > 0:
  37         real_cmd += ["-p", port]
  38
  39     host = gitConfig("git-p4.host")
  40     if len(host) > 0:
  41         real_cmd += ["-h", host]
  42
  43     client = gitConfig("git-p4.client")
  44     if len(client) > 0:
  45         real_cmd += ["-c", client]
  46
  47
  48     if isinstance(cmd,basestring):
  49         real_cmd = ' '.join(real_cmd) + ' ' + cmd
  50     else:
  51         real_cmd += cmd
  52     return real_cmd
  53
  54 def chdir(dir):
  55     # P4 uses the PWD environment variable rather than getcwd(). Since we're
  56     # not using the shell, we have to set it ourselves.  This path could
  57     # be relative, so go there first, then figure out where we ended up.
  58     os.chdir(dir)
  59     os.environ['PWD'] = os.getcwd()
  60
  61 def die(msg):
  62     if verbose:
  63         raise Exception(msg)
  64     else:
  65         sys.stderr.write(msg + "\n")
  66         sys.exit(1)
  67
  68 def write_pipe(c, stdin):
  69     if verbose:
  70         sys.stderr.write('Writing pipe: %s\n' % str(c))
  71
  72     expand = isinstance(c,basestring)
  73     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
  74     pipe = p.stdin
  75     val = pipe.write(stdin)
  76     pipe.close()
  77     if p.wait():
  78         die('Command failed: %s' % str(c))
  79
  80     return val
  81
  82 def p4_write_pipe(c, stdin):
  83     real_cmd = p4_build_cmd(c)
  84     return write_pipe(real_cmd, stdin)
  85
  86 def read_pipe(c, ignore_error=False):
  87     if verbose:
  88         sys.stderr.write('Reading pipe: %s\n' % str(c))
  89
  90     expand = isinstance(c,basestring)
  91     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
  92     pipe = p.stdout
  93     val = pipe.read()
  94     if p.wait() and not ignore_error:
  95         die('Command failed: %s' % str(c))
  96
  97     return val
  98
  99 def p4_read_pipe(c, ignore_error=False):
 100     real_cmd = p4_build_cmd(c)
 101     return read_pipe(real_cmd, ignore_error)
 102
 103 def read_pipe_lines(c):
 104     if verbose:
 105         sys.stderr.write('Reading pipe: %s\n' % str(c))
 106
 107     expand = isinstance(c, basestring)
 108     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
 109     pipe = p.stdout
 110     val = pipe.readlines()
 111     if pipe.close() or p.wait():
 112         die('Command failed: %s' % str(c))
 113
 114     return val
 115
 116 def p4_read_pipe_lines(c):
 117     """Specifically invoke p4 on the command supplied. """
 118     real_cmd = p4_build_cmd(c)
 119     return read_pipe_lines(real_cmd)
 120
 121 def system(cmd):
 122     expand = isinstance(cmd,basestring)
 123     if verbose:
 124         sys.stderr.write("executing %s\n" % str(cmd))
 125     subprocess.check_call(cmd, shell=expand)
 126
 127 def p4_system(cmd):
 128     """Specifically invoke p4 as the system command. """
 129     real_cmd = p4_build_cmd(cmd)
 130     expand = isinstance(real_cmd, basestring)
 131     subprocess.check_call(real_cmd, shell=expand)
 132
 133 def p4_integrate(src, dest):
 134     p4_system(["integrate", "-Dt", src, dest])
 135
 136 def p4_sync(path):
 137     p4_system(["sync", path])
 138
 139 def p4_add(f):
 140     p4_system(["add", f])
 141
 142 def p4_delete(f):
 143     p4_system(["delete", f])
 144
 145 def p4_edit(f):
 146     p4_system(["edit", f])
 147
 148 def p4_revert(f):
 149     p4_system(["revert", f])
 150
 151 def p4_reopen(type, file):
 152     p4_system(["reopen", "-t", type, file])
 153
 154 #
 155 # Canonicalize the p4 type and return a tuple of the
 156 # base type, plus any modifiers.  See "p4 help filetypes"
 157 # for a list and explanation.
 158 #
 159 def split_p4_type(p4type):
 160
 161     p4_filetypes_historical = {
 162         "ctempobj": "binary+Sw",
 163         "ctext": "text+C",
 164         "cxtext": "text+Cx",
 165         "ktext": "text+k",
 166         "kxtext": "text+kx",
 167         "ltext": "text+F",
 168         "tempobj": "binary+FSw",
 169         "ubinary": "binary+F",
 170         "uresource": "resource+F",
 171         "uxbinary": "binary+Fx",
 172         "xbinary": "binary+x",
 173         "xltext": "text+Fx",
 174         "xtempobj": "binary+Swx",
 175         "xtext": "text+x",
 176         "xunicode": "unicode+x",
 177         "xutf16": "utf16+x",
 178     }
 179     if p4type in p4_filetypes_historical:
 180         p4type = p4_filetypes_historical[p4type]
 181     mods = ""
 182     s = p4type.split("+")
 183     base = s[0]
 184     mods = ""
 185     if len(s) > 1:
 186         mods = s[1]
 187     return (base, mods)
 188
 189
 190 def setP4ExecBit(file, mode):
 191     # Reopens an already open file and changes the execute bit to match
 192     # the execute bit setting in the passed in mode.
 193
 194     p4Type = "+x"
 195
 196     if not isModeExec(mode):
 197         p4Type = getP4OpenedType(file)
 198         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
 199         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
 200         if p4Type[-1] == "+":
 201             p4Type = p4Type[0:-1]
 202
 203     p4_reopen(p4Type, file)
 204
 205 def getP4OpenedType(file):
 206     # Returns the perforce file type for the given file.
 207
 208     result = p4_read_pipe(["opened", file])
 209     match = re.match(".*\((.+)\)\r?$", result)
 210     if match:
 211         return match.group(1)
 212     else:
 213         die("Could not determine file type for %s (result: '%s')" % (file, result))
 214
 215 def diffTreePattern():
 216     # This is a simple generator for the diff tree regex pattern. This could be
 217     # a class variable if this and parseDiffTreeEntry were a part of a class.
 218     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
 219     while True:
 220         yield pattern
 221
 222 def parseDiffTreeEntry(entry):
 223     """Parses a single diff tree entry into its component elements.
 224
 225     See git-diff-tree(1) manpage for details about the format of the diff
 226     output. This method returns a dictionary with the following elements:
 227
 228     src_mode - The mode of the source file
 229     dst_mode - The mode of the destination file
 230     src_sha1 - The sha1 for the source file
 231     dst_sha1 - The sha1 fr the destination file
 232     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
 233     status_score - The score for the status (applicable for 'C' and 'R'
 234                    statuses). This is None if there is no score.
 235     src - The path for the source file.
 236     dst - The path for the destination file. This is only present for
 237           copy or renames. If it is not present, this is None.
 238
 239     If the pattern is not matched, None is returned."""
 240
 241     match = diffTreePattern().next().match(entry)
 242     if match:
 243         return {
 244             'src_mode': match.group(1),
 245             'dst_mode': match.group(2),
 246             'src_sha1': match.group(3),
 247             'dst_sha1': match.group(4),
 248             'status': match.group(5),
 249             'status_score': match.group(6),
 250             'src': match.group(7),
 251             'dst': match.group(10)
 252         }
 253     return None
 254
 255 def isModeExec(mode):
 256     # Returns True if the given git mode represents an executable file,
 257     # otherwise False.
 258     return mode[-3:] == "755"
 259
 260 def isModeExecChanged(src_mode, dst_mode):
 261     return isModeExec(src_mode) != isModeExec(dst_mode)
 262
 263 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
 264
 265     if isinstance(cmd,basestring):
 266         cmd = "-G " + cmd
 267         expand = True
 268     else:
 269         cmd = ["-G"] + cmd
 270         expand = False
 271
 272     cmd = p4_build_cmd(cmd)
 273     if verbose:
 274         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
 275
 276     # Use a temporary file to avoid deadlocks without
 277     # subprocess.communicate(), which would put another copy
 278     # of stdout into memory.
 279     stdin_file = None
 280     if stdin is not None:
 281         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
 282         if isinstance(stdin,basestring):
 283             stdin_file.write(stdin)
 284         else:
 285             for i in stdin:
 286                 stdin_file.write(i + '\n')
 287         stdin_file.flush()
 288         stdin_file.seek(0)
 289
 290     p4 = subprocess.Popen(cmd,
 291                           shell=expand,
 292                           stdin=stdin_file,
 293                           stdout=subprocess.PIPE)
 294
 295     result = []
 296     try:
 297         while True:
 298             entry = marshal.load(p4.stdout)
 299             if cb is not None:
 300                 cb(entry)
 301             else:
 302                 result.append(entry)
 303     except EOFError:
 304         pass
 305     exitCode = p4.wait()
 306     if exitCode != 0:
 307         entry = {}
 308         entry["p4ExitCode"] = exitCode
 309         result.append(entry)
 310
 311     return result
 312
 313 def p4Cmd(cmd):
 314     list = p4CmdList(cmd)
 315     result = {}
 316     for entry in list:
 317         result.update(entry)
 318     return result;
 319
 320 def p4Where(depotPath):
 321     if not depotPath.endswith("/"):
 322         depotPath += "/"
 323     depotPath = depotPath + "..."
 324     outputList = p4CmdList(["where", depotPath])
 325     output = None
 326     for entry in outputList:
 327         if "depotFile" in entry:
 328             if entry["depotFile"] == depotPath:
 329                 output = entry
 330                 break
 331         elif "data" in entry:
 332             data = entry.get("data")
 333             space = data.find(" ")
 334             if data[:space] == depotPath:
 335                 output = entry
 336                 break
 337     if output == None:
 338         return ""
 339     if output["code"] == "error":
 340         return ""
 341     clientPath = ""
 342     if "path" in output:
 343         clientPath = output.get("path")
 344     elif "data" in output:
 345         data = output.get("data")
 346         lastSpace = data.rfind(" ")
 347         clientPath = data[lastSpace + 1:]
 348
 349     if clientPath.endswith("..."):
 350         clientPath = clientPath[:-3]
 351     return clientPath
 352
 353 def currentGitBranch():
 354     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
 355
 356 def isValidGitDir(path):
 357     if (os.path.exists(path + "/HEAD")
 358         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
 359         return True;
 360     return False
 361
 362 def parseRevision(ref):
 363     return read_pipe("git rev-parse %s" % ref).strip()
 364
 365 def branchExists(ref):
 366     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
 367                      ignore_error=True)
 368     return len(rev) > 0
 369
 370 def extractLogMessageFromGitCommit(commit):
 371     logMessage = ""
 372
 373     ## fixme: title is first line of commit, not 1st paragraph.
 374     foundTitle = False
 375     for log in read_pipe_lines("git cat-file commit %s" % commit):
 376        if not foundTitle:
 377            if len(log) == 1:
 378                foundTitle = True
 379            continue
 380
 381        logMessage += log
 382     return logMessage
 383
 384 def extractSettingsGitLog(log):
 385     values = {}
 386     for line in log.split("\n"):
 387         line = line.strip()
 388         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
 389         if not m:
 390             continue
 391
 392         assignments = m.group(1).split (':')
 393         for a in assignments:
 394             vals = a.split ('=')
 395             key = vals[0].strip()
 396             val = ('='.join (vals[1:])).strip()
 397             if val.endswith ('\"') and val.startswith('"'):
 398                 val = val[1:-1]
 399
 400             values[key] = val
 401
 402     paths = values.get("depot-paths")
 403     if not paths:
 404         paths = values.get("depot-path")
 405     if paths:
 406         values['depot-paths'] = paths.split(',')
 407     return values
 408
 409 def gitBranchExists(branch):
 410     proc = subprocess.Popen(["git", "rev-parse", branch],
 411                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
 412     return proc.wait() == 0;
 413
 414 _gitConfig = {}
 415 def gitConfig(key, args = None): # set args to "--bool", for instance
 416     if not _gitConfig.has_key(key):
 417         argsFilter = ""
 418         if args != None:
 419             argsFilter = "%s " % args
 420         cmd = "git config %s%s" % (argsFilter, key)
 421         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
 422     return _gitConfig[key]
 423
 424 def gitConfigList(key):
 425     if not _gitConfig.has_key(key):
 426         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
 427     return _gitConfig[key]
 428
 429 def p4BranchesInGit(branchesAreInRemotes = True):
 430     branches = {}
 431
 432     cmdline = "git rev-parse --symbolic "
 433     if branchesAreInRemotes:
 434         cmdline += " --remotes"
 435     else:
 436         cmdline += " --branches"
 437
 438     for line in read_pipe_lines(cmdline):
 439         line = line.strip()
 440
 441         ## only import to p4/
 442         if not line.startswith('p4/') or line == "p4/HEAD":
 443             continue
 444         branch = line
 445
 446         # strip off p4
 447         branch = re.sub ("^p4/", "", line)
 448
 449         branches[branch] = parseRevision(line)
 450     return branches
 451
 452 def findUpstreamBranchPoint(head = "HEAD"):
 453     branches = p4BranchesInGit()
 454     # map from depot-path to branch name
 455     branchByDepotPath = {}
 456     for branch in branches.keys():
 457         tip = branches[branch]
 458         log = extractLogMessageFromGitCommit(tip)
 459         settings = extractSettingsGitLog(log)
 460         if settings.has_key("depot-paths"):
 461             paths = ",".join(settings["depot-paths"])
 462             branchByDepotPath[paths] = "remotes/p4/" + branch
 463
 464     settings = None
 465     parent = 0
 466     while parent < 65535:
 467         commit = head + "~%s" % parent
 468         log = extractLogMessageFromGitCommit(commit)
 469         settings = extractSettingsGitLog(log)
 470         if settings.has_key("depot-paths"):
 471             paths = ",".join(settings["depot-paths"])
 472             if branchByDepotPath.has_key(paths):
 473                 return [branchByDepotPath[paths], settings]
 474
 475         parent = parent + 1
 476
 477     return ["", settings]
 478
 479 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
 480     if not silent:
 481         print ("Creating/updating branch(es) in %s based on origin branch(es)"
 482                % localRefPrefix)
 483
 484     originPrefix = "origin/p4/"
 485
 486     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
 487         line = line.strip()
 488         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
 489             continue
 490
 491         headName = line[len(originPrefix):]
 492         remoteHead = localRefPrefix + headName
 493         originHead = line
 494
 495         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
 496         if (not original.has_key('depot-paths')
 497             or not original.has_key('change')):
 498             continue
 499
 500         update = False
 501         if not gitBranchExists(remoteHead):
 502             if verbose:
 503                 print "creating %s" % remoteHead
 504             update = True
 505         else:
 506             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
 507             if settings.has_key('change') > 0:
 508                 if settings['depot-paths'] == original['depot-paths']:
 509                     originP4Change = int(original['change'])
 510                     p4Change = int(settings['change'])
 511                     if originP4Change > p4Change:
 512                         print ("%s (%s) is newer than %s (%s). "
 513                                "Updating p4 branch from origin."
 514                                % (originHead, originP4Change,
 515                                   remoteHead, p4Change))
 516                         update = True
 517                 else:
 518                     print ("Ignoring: %s was imported from %s while "
 519                            "%s was imported from %s"
 520                            % (originHead, ','.join(original['depot-paths']),
 521                               remoteHead, ','.join(settings['depot-paths'])))
 522
 523         if update:
 524             system("git update-ref %s %s" % (remoteHead, originHead))
 525
 526 def originP4BranchesExist():
 527         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
 528
 529 def p4ChangesForPaths(depotPaths, changeRange):
 530     assert depotPaths
 531     cmd = ['changes']
 532     for p in depotPaths:
 533         cmd += ["%s...%s" % (p, changeRange)]
 534     output = p4_read_pipe_lines(cmd)
 535
 536     changes = {}
 537     for line in output:
 538         changeNum = int(line.split(" ")[1])
 539         changes[changeNum] = True
 540
 541     changelist = changes.keys()
 542     changelist.sort()
 543     return changelist
 544
 545 def p4PathStartsWith(path, prefix):
 546     # This method tries to remedy a potential mixed-case issue:
 547     #
 548     # If UserA adds  //depot/DirA/file1
 549     # and UserB adds //depot/dira/file2
 550     #
 551     # we may or may not have a problem. If you have core.ignorecase=true,
 552     # we treat DirA and dira as the same directory
 553     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
 554     if ignorecase:
 555         return path.lower().startswith(prefix.lower())
 556     return path.startswith(prefix)
 557
 558 class Command:
 559     def __init__(self):
 560         self.usage = "usage: %prog [options]"
 561         self.needsGit = True
 562
 563 class P4UserMap:
 564     def __init__(self):
 565         self.userMapFromPerforceServer = False
 566
 567     def getUserCacheFilename(self):
 568         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
 569         return home + "/.gitp4-usercache.txt"
 570
 571     def getUserMapFromPerforceServer(self):
 572         if self.userMapFromPerforceServer:
 573             return
 574         self.users = {}
 575         self.emails = {}
 576
 577         for output in p4CmdList("users"):
 578             if not output.has_key("User"):
 579                 continue
 580             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 581             self.emails[output["Email"]] = output["User"]
 582
 583
 584         s = ''
 585         for (key, val) in self.users.items():
 586             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
 587
 588         open(self.getUserCacheFilename(), "wb").write(s)
 589         self.userMapFromPerforceServer = True
 590
 591     def loadUserMapFromCache(self):
 592         self.users = {}
 593         self.userMapFromPerforceServer = False
 594         try:
 595             cache = open(self.getUserCacheFilename(), "rb")
 596             lines = cache.readlines()
 597             cache.close()
 598             for line in lines:
 599                 entry = line.strip().split("\t")
 600                 self.users[entry[0]] = entry[1]
 601         except IOError:
 602             self.getUserMapFromPerforceServer()
 603
 604 class P4Debug(Command):
 605     def __init__(self):
 606         Command.__init__(self)
 607         self.options = [
 608             optparse.make_option("--verbose", dest="verbose", action="store_true",
 609                                  default=False),
 610             ]
 611         self.description = "A tool to debug the output of p4 -G."
 612         self.needsGit = False
 613         self.verbose = False
 614
 615     def run(self, args):
 616         j = 0
 617         for output in p4CmdList(args):
 618             print 'Element: %d' % j
 619             j += 1
 620             print output
 621         return True
 622
 623 class P4RollBack(Command):
 624     def __init__(self):
 625         Command.__init__(self)
 626         self.options = [
 627             optparse.make_option("--verbose", dest="verbose", action="store_true"),
 628             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
 629         ]
 630         self.description = "A tool to debug the multi-branch import. Don't use :)"
 631         self.verbose = False
 632         self.rollbackLocalBranches = False
 633
 634     def run(self, args):
 635         if len(args) != 1:
 636             return False
 637         maxChange = int(args[0])
 638
 639         if "p4ExitCode" in p4Cmd("changes -m 1"):
 640             die("Problems executing p4");
 641
 642         if self.rollbackLocalBranches:
 643             refPrefix = "refs/heads/"
 644             lines = read_pipe_lines("git rev-parse --symbolic --branches")
 645         else:
 646             refPrefix = "refs/remotes/"
 647             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
 648
 649         for line in lines:
 650             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
 651                 line = line.strip()
 652                 ref = refPrefix + line
 653                 log = extractLogMessageFromGitCommit(ref)
 654                 settings = extractSettingsGitLog(log)
 655
 656                 depotPaths = settings['depot-paths']
 657                 change = settings['change']
 658
 659                 changed = False
 660
 661                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
 662                                                            for p in depotPaths]))) == 0:
 663                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
 664                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
 665                     continue
 666
 667                 while change and int(change) > maxChange:
 668                     changed = True
 669                     if self.verbose:
 670                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
 671                     system("git update-ref %s \"%s^\"" % (ref, ref))
 672                     log = extractLogMessageFromGitCommit(ref)
 673                     settings =  extractSettingsGitLog(log)
 674
 675
 676                     depotPaths = settings['depot-paths']
 677                     change = settings['change']
 678
 679                 if changed:
 680                     print "%s rewound to %s" % (ref, change)
 681
 682         return True
 683
 684 class P4Submit(Command, P4UserMap):
 685     def __init__(self):
 686         Command.__init__(self)
 687         P4UserMap.__init__(self)
 688         self.options = [
 689                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
 690                 optparse.make_option("--origin", dest="origin"),
 691                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
 692                 # preserve the user, requires relevant p4 permissions
 693                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
 694         ]
 695         self.description = "Submit changes from git to the perforce depot."
 696         self.usage += " [name of git branch to submit into perforce depot]"
 697         self.interactive = True
 698         self.origin = ""
 699         self.detectRenames = False
 700         self.verbose = False
 701         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
 702         self.isWindows = (platform.system() == "Windows")
 703         self.myP4UserId = None
 704
 705     def check(self):
 706         if len(p4CmdList("opened ...")) > 0:
 707             die("You have files opened with perforce! Close them before starting the sync.")
 708
 709     # replaces everything between 'Description:' and the next P4 submit template field with the
 710     # commit message
 711     def prepareLogMessage(self, template, message):
 712         result = ""
 713
 714         inDescriptionSection = False
 715
 716         for line in template.split("\n"):
 717             if line.startswith("#"):
 718                 result += line + "\n"
 719                 continue
 720
 721             if inDescriptionSection:
 722                 if line.startswith("Files:") or line.startswith("Jobs:"):
 723                     inDescriptionSection = False
 724                 else:
 725                     continue
 726             else:
 727                 if line.startswith("Description:"):
 728                     inDescriptionSection = True
 729                     line += "\n"
 730                     for messageLine in message.split("\n"):
 731                         line += "\t" + messageLine + "\n"
 732
 733             result += line + "\n"
 734
 735         return result
 736
 737     def p4UserForCommit(self,id):
 738         # Return the tuple (perforce user,git email) for a given git commit id
 739         self.getUserMapFromPerforceServer()
 740         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
 741         gitEmail = gitEmail.strip()
 742         if not self.emails.has_key(gitEmail):
 743             return (None,gitEmail)
 744         else:
 745             return (self.emails[gitEmail],gitEmail)
 746
 747     def checkValidP4Users(self,commits):
 748         # check if any git authors cannot be mapped to p4 users
 749         for id in commits:
 750             (user,email) = self.p4UserForCommit(id)
 751             if not user:
 752                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
 753                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
 754                     print "%s" % msg
 755                 else:
 756                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
 757
 758     def lastP4Changelist(self):
 759         # Get back the last changelist number submitted in this client spec. This
 760         # then gets used to patch up the username in the change. If the same
 761         # client spec is being used by multiple processes then this might go
 762         # wrong.
 763         results = p4CmdList("client -o")        # find the current client
 764         client = None
 765         for r in results:
 766             if r.has_key('Client'):
 767                 client = r['Client']
 768                 break
 769         if not client:
 770             die("could not get client spec")
 771         results = p4CmdList(["changes", "-c", client, "-m", "1"])
 772         for r in results:
 773             if r.has_key('change'):
 774                 return r['change']
 775         die("Could not get changelist number for last submit - cannot patch up user details")
 776
 777     def modifyChangelistUser(self, changelist, newUser):
 778         # fixup the user field of a changelist after it has been submitted.
 779         changes = p4CmdList("change -o %s" % changelist)
 780         if len(changes) != 1:
 781             die("Bad output from p4 change modifying %s to user %s" %
 782                 (changelist, newUser))
 783
 784         c = changes[0]
 785         if c['User'] == newUser: return   # nothing to do
 786         c['User'] = newUser
 787         input = marshal.dumps(c)
 788
 789         result = p4CmdList("change -f -i", stdin=input)
 790         for r in result:
 791             if r.has_key('code'):
 792                 if r['code'] == 'error':
 793                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
 794             if r.has_key('data'):
 795                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
 796                 return
 797         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
 798
 799     def canChangeChangelists(self):
 800         # check to see if we have p4 admin or super-user permissions, either of
 801         # which are required to modify changelists.
 802         results = p4CmdList("protects %s" % self.depotPath)
 803         for r in results:
 804             if r.has_key('perm'):
 805                 if r['perm'] == 'admin':
 806                     return 1
 807                 if r['perm'] == 'super':
 808                     return 1
 809         return 0
 810
 811     def p4UserId(self):
 812         if self.myP4UserId:
 813             return self.myP4UserId
 814
 815         results = p4CmdList("user -o")
 816         for r in results:
 817             if r.has_key('User'):
 818                 self.myP4UserId = r['User']
 819                 return r['User']
 820         die("Could not find your p4 user id")
 821
 822     def p4UserIsMe(self, p4User):
 823         # return True if the given p4 user is actually me
 824         me = self.p4UserId()
 825         if not p4User or p4User != me:
 826             return False
 827         else:
 828             return True
 829
 830     def prepareSubmitTemplate(self):
 831         # remove lines in the Files section that show changes to files outside the depot path we're committing into
 832         template = ""
 833         inFilesSection = False
 834         for line in p4_read_pipe_lines(['change', '-o']):
 835             if line.endswith("\r\n"):
 836                 line = line[:-2] + "\n"
 837             if inFilesSection:
 838                 if line.startswith("\t"):
 839                     # path starts and ends with a tab
 840                     path = line[1:]
 841                     lastTab = path.rfind("\t")
 842                     if lastTab != -1:
 843                         path = path[:lastTab]
 844                         if not p4PathStartsWith(path, self.depotPath):
 845                             continue
 846                 else:
 847                     inFilesSection = False
 848             else:
 849                 if line.startswith("Files:"):
 850                     inFilesSection = True
 851
 852             template += line
 853
 854         return template
 855
 856     def edit_template(self, template_file):
 857         """Invoke the editor to let the user change the submission
 858            message.  Return true if okay to continue with the submit."""
 859
 860         # if configured to skip the editing part, just submit
 861         if gitConfig("git-p4.skipSubmitEdit") == "true":
 862             return True
 863
 864         # look at the modification time, to check later if the user saved
 865         # the file
 866         mtime = os.stat(template_file).st_mtime
 867
 868         # invoke the editor
 869         if os.environ.has_key("P4EDITOR"):
 870             editor = os.environ.get("P4EDITOR")
 871         else:
 872             editor = read_pipe("git var GIT_EDITOR").strip()
 873         system(editor + " " + template_file)
 874
 875         # If the file was not saved, prompt to see if this patch should
 876         # be skipped.  But skip this verification step if configured so.
 877         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
 878             return True
 879
 880         # modification time updated means user saved the file
 881         if os.stat(template_file).st_mtime > mtime:
 882             return True
 883
 884         while True:
 885             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
 886             if response == 'y':
 887                 return True
 888             if response == 'n':
 889                 return False
 890
 891     def applyCommit(self, id):
 892         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
 893
 894         (p4User, gitEmail) = self.p4UserForCommit(id)
 895
 896         if not self.detectRenames:
 897             # If not explicitly set check the config variable
 898             self.detectRenames = gitConfig("git-p4.detectRenames")
 899
 900         if self.detectRenames.lower() == "false" or self.detectRenames == "":
 901             diffOpts = ""
 902         elif self.detectRenames.lower() == "true":
 903             diffOpts = "-M"
 904         else:
 905             diffOpts = "-M%s" % self.detectRenames
 906
 907         detectCopies = gitConfig("git-p4.detectCopies")
 908         if detectCopies.lower() == "true":
 909             diffOpts += " -C"
 910         elif detectCopies != "" and detectCopies.lower() != "false":
 911             diffOpts += " -C%s" % detectCopies
 912
 913         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
 914             diffOpts += " --find-copies-harder"
 915
 916         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
 917         filesToAdd = set()
 918         filesToDelete = set()
 919         editedFiles = set()
 920         filesToChangeExecBit = {}
 921         for line in diff:
 922             diff = parseDiffTreeEntry(line)
 923             modifier = diff['status']
 924             path = diff['src']
 925             if modifier == "M":
 926                 p4_edit(path)
 927                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 928                     filesToChangeExecBit[path] = diff['dst_mode']
 929                 editedFiles.add(path)
 930             elif modifier == "A":
 931                 filesToAdd.add(path)
 932                 filesToChangeExecBit[path] = diff['dst_mode']
 933                 if path in filesToDelete:
 934                     filesToDelete.remove(path)
 935             elif modifier == "D":
 936                 filesToDelete.add(path)
 937                 if path in filesToAdd:
 938                     filesToAdd.remove(path)
 939             elif modifier == "C":
 940                 src, dest = diff['src'], diff['dst']
 941                 p4_integrate(src, dest)
 942                 if diff['src_sha1'] != diff['dst_sha1']:
 943                     p4_edit(dest)
 944                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 945                     p4_edit(dest)
 946                     filesToChangeExecBit[dest] = diff['dst_mode']
 947                 os.unlink(dest)
 948                 editedFiles.add(dest)
 949             elif modifier == "R":
 950                 src, dest = diff['src'], diff['dst']
 951                 p4_integrate(src, dest)
 952                 if diff['src_sha1'] != diff['dst_sha1']:
 953                     p4_edit(dest)
 954                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
 955                     p4_edit(dest)
 956                     filesToChangeExecBit[dest] = diff['dst_mode']
 957                 os.unlink(dest)
 958                 editedFiles.add(dest)
 959                 filesToDelete.add(src)
 960             else:
 961                 die("unknown modifier %s for %s" % (modifier, path))
 962
 963         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
 964         patchcmd = diffcmd + " | git apply "
 965         tryPatchCmd = patchcmd + "--check -"
 966         applyPatchCmd = patchcmd + "--check --apply -"
 967
 968         if os.system(tryPatchCmd) != 0:
 969             print "Unfortunately applying the change failed!"
 970             print "What do you want to do?"
 971             response = "x"
 972             while response != "s" and response != "a" and response != "w":
 973                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
 974                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
 975             if response == "s":
 976                 print "Skipping! Good luck with the next patches..."
 977                 for f in editedFiles:
 978                     p4_revert(f)
 979                 for f in filesToAdd:
 980                     os.remove(f)
 981                 return
 982             elif response == "a":
 983                 os.system(applyPatchCmd)
 984                 if len(filesToAdd) > 0:
 985                     print "You may also want to call p4 add on the following files:"
 986                     print " ".join(filesToAdd)
 987                 if len(filesToDelete):
 988                     print "The following files should be scheduled for deletion with p4 delete:"
 989                     print " ".join(filesToDelete)
 990                 die("Please resolve and submit the conflict manually and "
 991                     + "continue afterwards with git-p4 submit --continue")
 992             elif response == "w":
 993                 system(diffcmd + " > patch.txt")
 994                 print "Patch saved to patch.txt in %s !" % self.clientPath
 995                 die("Please resolve and submit the conflict manually and "
 996                     "continue afterwards with git-p4 submit --continue")
 997
 998         system(applyPatchCmd)
 999
1000         for f in filesToAdd:
1001             p4_add(f)
1002         for f in filesToDelete:
1003             p4_revert(f)
1004             p4_delete(f)
1005
1006         # Set/clear executable bits
1007         for f in filesToChangeExecBit.keys():
1008             mode = filesToChangeExecBit[f]
1009             setP4ExecBit(f, mode)
1010
1011         logMessage = extractLogMessageFromGitCommit(id)
1012         logMessage = logMessage.strip()
1013
1014         template = self.prepareSubmitTemplate()
1015
1016         if self.interactive:
1017             submitTemplate = self.prepareLogMessage(template, logMessage)
1018
1019             if self.preserveUser:
1020                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1021
1022             if os.environ.has_key("P4DIFF"):
1023                 del(os.environ["P4DIFF"])
1024             diff = ""
1025             for editedFile in editedFiles:
1026                 diff += p4_read_pipe(['diff', '-du', editedFile])
1027
1028             newdiff = ""
1029             for newFile in filesToAdd:
1030                 newdiff += "==== new file ====\n"
1031                 newdiff += "--- /dev/null\n"
1032                 newdiff += "+++ %s\n" % newFile
1033                 f = open(newFile, "r")
1034                 for line in f.readlines():
1035                     newdiff += "+" + line
1036                 f.close()
1037
1038             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1039                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1040                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1041                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1042
1043             separatorLine = "######## everything below this line is just the diff #######\n"
1044
1045             (handle, fileName) = tempfile.mkstemp()
1046             tmpFile = os.fdopen(handle, "w+")
1047             if self.isWindows:
1048                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1049                 separatorLine = separatorLine.replace("\n", "\r\n")
1050                 newdiff = newdiff.replace("\n", "\r\n")
1051             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1052             tmpFile.close()
1053
1054             if self.edit_template(fileName):
1055                 # read the edited message and submit
1056                 tmpFile = open(fileName, "rb")
1057                 message = tmpFile.read()
1058                 tmpFile.close()
1059                 submitTemplate = message[:message.index(separatorLine)]
1060                 if self.isWindows:
1061                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1062                 p4_write_pipe(['submit', '-i'], submitTemplate)
1063
1064                 if self.preserveUser:
1065                     if p4User:
1066                         # Get last changelist number. Cannot easily get it from
1067                         # the submit command output as the output is
1068                         # unmarshalled.
1069                         changelist = self.lastP4Changelist()
1070                         self.modifyChangelistUser(changelist, p4User)
1071             else:
1072                 # skip this patch
1073                 print "Submission cancelled, undoing p4 changes."
1074                 for f in editedFiles:
1075                     p4_revert(f)
1076                 for f in filesToAdd:
1077                     p4_revert(f)
1078                     os.remove(f)
1079
1080             os.remove(fileName)
1081         else:
1082             fileName = "submit.txt"
1083             file = open(fileName, "w+")
1084             file.write(self.prepareLogMessage(template, logMessage))
1085             file.close()
1086             print ("Perforce submit template written as %s. "
1087                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1088                    % (fileName, fileName))
1089
1090     def run(self, args):
1091         if len(args) == 0:
1092             self.master = currentGitBranch()
1093             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1094                 die("Detecting current git branch failed!")
1095         elif len(args) == 1:
1096             self.master = args[0]
1097             if not branchExists(self.master):
1098                 die("Branch %s does not exist" % self.master)
1099         else:
1100             return False
1101
1102         allowSubmit = gitConfig("git-p4.allowSubmit")
1103         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1104             die("%s is not in git-p4.allowSubmit" % self.master)
1105
1106         [upstream, settings] = findUpstreamBranchPoint()
1107         self.depotPath = settings['depot-paths'][0]
1108         if len(self.origin) == 0:
1109             self.origin = upstream
1110
1111         if self.preserveUser:
1112             if not self.canChangeChangelists():
1113                 die("Cannot preserve user names without p4 super-user or admin permissions")
1114
1115         if self.verbose:
1116             print "Origin branch is " + self.origin
1117
1118         if len(self.depotPath) == 0:
1119             print "Internal error: cannot locate perforce depot path from existing branches"
1120             sys.exit(128)
1121
1122         self.clientPath = p4Where(self.depotPath)
1123
1124         if len(self.clientPath) == 0:
1125             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1126             sys.exit(128)
1127
1128         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1129         self.oldWorkingDirectory = os.getcwd()
1130
1131         # ensure the clientPath exists
1132         if not os.path.exists(self.clientPath):
1133             os.makedirs(self.clientPath)
1134
1135         chdir(self.clientPath)
1136         print "Synchronizing p4 checkout..."
1137         p4_sync("...")
1138         self.check()
1139
1140         commits = []
1141         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1142             commits.append(line.strip())
1143         commits.reverse()
1144
1145         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1146             self.checkAuthorship = False
1147         else:
1148             self.checkAuthorship = True
1149
1150         if self.preserveUser:
1151             self.checkValidP4Users(commits)
1152
1153         while len(commits) > 0:
1154             commit = commits[0]
1155             commits = commits[1:]
1156             self.applyCommit(commit)
1157             if not self.interactive:
1158                 break
1159
1160         if len(commits) == 0:
1161             print "All changes applied!"
1162             chdir(self.oldWorkingDirectory)
1163
1164             sync = P4Sync()
1165             sync.run([])
1166
1167             rebase = P4Rebase()
1168             rebase.rebase()
1169
1170         return True
1171
1172 class P4Sync(Command, P4UserMap):
1173     delete_actions = ( "delete", "move/delete", "purge" )
1174
1175     def __init__(self):
1176         Command.__init__(self)
1177         P4UserMap.__init__(self)
1178         self.options = [
1179                 optparse.make_option("--branch", dest="branch"),
1180                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1181                 optparse.make_option("--changesfile", dest="changesFile"),
1182                 optparse.make_option("--silent", dest="silent", action="store_true"),
1183                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1184                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1185                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1186                                      help="Import into refs/heads/ , not refs/remotes"),
1187                 optparse.make_option("--max-changes", dest="maxChanges"),
1188                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1189                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1190                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1191                                      help="Only sync files that are included in the Perforce Client Spec")
1192         ]
1193         self.description = """Imports from Perforce into a git repository.\n
1194     example:
1195     //depot/my/project/ -- to import the current head
1196     //depot/my/project/@all -- to import everything
1197     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1198
1199     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1200
1201         self.usage += " //depot/path[@revRange]"
1202         self.silent = False
1203         self.createdBranches = set()
1204         self.committedChanges = set()
1205         self.branch = ""
1206         self.detectBranches = False
1207         self.detectLabels = False
1208         self.changesFile = ""
1209         self.syncWithOrigin = True
1210         self.verbose = False
1211         self.importIntoRemotes = True
1212         self.maxChanges = ""
1213         self.isWindows = (platform.system() == "Windows")
1214         self.keepRepoPath = False
1215         self.depotPaths = None
1216         self.p4BranchesInGit = []
1217         self.cloneExclude = []
1218         self.useClientSpec = False
1219         self.clientSpecDirs = []
1220
1221         if gitConfig("git-p4.syncFromOrigin") == "false":
1222             self.syncWithOrigin = False
1223
1224     #
1225     # P4 wildcards are not allowed in filenames.  P4 complains
1226     # if you simply add them, but you can force it with "-f", in
1227     # which case it translates them into %xx encoding internally.
1228     # Search for and fix just these four characters.  Do % last so
1229     # that fixing it does not inadvertently create new %-escapes.
1230     #
1231     def wildcard_decode(self, path):
1232         # Cannot have * in a filename in windows; untested as to
1233         # what p4 would do in such a case.
1234         if not self.isWindows:
1235             path = path.replace("%2A", "*")
1236         path = path.replace("%23", "#") \
1237                    .replace("%40", "@") \
1238                    .replace("%25", "%")
1239         return path
1240
1241     def extractFilesFromCommit(self, commit):
1242         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1243                              for path in self.cloneExclude]
1244         files = []
1245         fnum = 0
1246         while commit.has_key("depotFile%s" % fnum):
1247             path =  commit["depotFile%s" % fnum]
1248
1249             if [p for p in self.cloneExclude
1250                 if p4PathStartsWith(path, p)]:
1251                 found = False
1252             else:
1253                 found = [p for p in self.depotPaths
1254                          if p4PathStartsWith(path, p)]
1255             if not found:
1256                 fnum = fnum + 1
1257                 continue
1258
1259             file = {}
1260             file["path"] = path
1261             file["rev"] = commit["rev%s" % fnum]
1262             file["action"] = commit["action%s" % fnum]
1263             file["type"] = commit["type%s" % fnum]
1264             files.append(file)
1265             fnum = fnum + 1
1266         return files
1267
1268     def stripRepoPath(self, path, prefixes):
1269         if self.useClientSpec:
1270
1271             # if using the client spec, we use the output directory
1272             # specified in the client.  For example, a view
1273             #   //depot/foo/branch/... //client/branch/foo/...
1274             # will end up putting all foo/branch files into
1275             #  branch/foo/
1276             for val in self.clientSpecDirs:
1277                 if path.startswith(val[0]):
1278                     # replace the depot path with the client path
1279                     path = path.replace(val[0], val[1][1])
1280                     # now strip out the client (//client/...)
1281                     path = re.sub("^(//[^/]+/)", '', path)
1282                     # the rest is all path
1283                     return path
1284
1285         if self.keepRepoPath:
1286             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1287
1288         for p in prefixes:
1289             if p4PathStartsWith(path, p):
1290                 path = path[len(p):]
1291
1292         return path
1293
1294     def splitFilesIntoBranches(self, commit):
1295         branches = {}
1296         fnum = 0
1297         while commit.has_key("depotFile%s" % fnum):
1298             path =  commit["depotFile%s" % fnum]
1299             found = [p for p in self.depotPaths
1300                      if p4PathStartsWith(path, p)]
1301             if not found:
1302                 fnum = fnum + 1
1303                 continue
1304
1305             file = {}
1306             file["path"] = path
1307             file["rev"] = commit["rev%s" % fnum]
1308             file["action"] = commit["action%s" % fnum]
1309             file["type"] = commit["type%s" % fnum]
1310             fnum = fnum + 1
1311
1312             relPath = self.stripRepoPath(path, self.depotPaths)
1313
1314             for branch in self.knownBranches.keys():
1315
1316                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1317                 if relPath.startswith(branch + "/"):
1318                     if branch not in branches:
1319                         branches[branch] = []
1320                     branches[branch].append(file)
1321                     break
1322
1323         return branches
1324
1325     # output one file from the P4 stream
1326     # - helper for streamP4Files
1327
1328     def streamOneP4File(self, file, contents):
1329         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1330         relPath = self.wildcard_decode(relPath)
1331         if verbose:
1332             sys.stderr.write("%s\n" % relPath)
1333
1334         (type_base, type_mods) = split_p4_type(file["type"])
1335
1336         git_mode = "100644"
1337         if "x" in type_mods:
1338             git_mode = "100755"
1339         if type_base == "symlink":
1340             git_mode = "120000"
1341             # p4 print on a symlink contains "target\n"; remove the newline
1342             data = ''.join(contents)
1343             contents = [data[:-1]]
1344
1345         if type_base == "utf16":
1346             # p4 delivers different text in the python output to -G
1347             # than it does when using "print -o", or normal p4 client
1348             # operations.  utf16 is converted to ascii or utf8, perhaps.
1349             # But ascii text saved as -t utf16 is completely mangled.
1350             # Invoke print -o to get the real contents.
1351             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1352             contents = [ text ]
1353
1354         if type_base == "apple":
1355             # Apple filetype files will be streamed as a concatenation of
1356             # its appledouble header and the contents.  This is useless
1357             # on both macs and non-macs.  If using "print -q -o xx", it
1358             # will create "xx" with the data, and "%xx" with the header.
1359             # This is also not very useful.
1360             #
1361             # Ideally, someday, this script can learn how to generate
1362             # appledouble files directly and import those to git, but
1363             # non-mac machines can never find a use for apple filetype.
1364             print "\nIgnoring apple filetype file %s" % file['depotFile']
1365             return
1366
1367         # Perhaps windows wants unicode, utf16 newlines translated too;
1368         # but this is not doing it.
1369         if self.isWindows and type_base == "text":
1370             mangled = []
1371             for data in contents:
1372                 data = data.replace("\r\n", "\n")
1373                 mangled.append(data)
1374             contents = mangled
1375
1376         # Note that we do not try to de-mangle keywords on utf16 files,
1377         # even though in theory somebody may want that.
1378         if type_base in ("text", "unicode", "binary"):
1379             if "ko" in type_mods:
1380                 text = ''.join(contents)
1381                 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1382                 contents = [ text ]
1383             elif "k" in type_mods:
1384                 text = ''.join(contents)
1385                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1386                 contents = [ text ]
1387
1388         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1389
1390         # total length...
1391         length = 0
1392         for d in contents:
1393             length = length + len(d)
1394
1395         self.gitStream.write("data %d\n" % length)
1396         for d in contents:
1397             self.gitStream.write(d)
1398         self.gitStream.write("\n")
1399
1400     def streamOneP4Deletion(self, file):
1401         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1402         if verbose:
1403             sys.stderr.write("delete %s\n" % relPath)
1404         self.gitStream.write("D %s\n" % relPath)
1405
1406     # handle another chunk of streaming data
1407     def streamP4FilesCb(self, marshalled):
1408
1409         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1410             # start of a new file - output the old one first
1411             self.streamOneP4File(self.stream_file, self.stream_contents)
1412             self.stream_file = {}
1413             self.stream_contents = []
1414             self.stream_have_file_info = False
1415
1416         # pick up the new file information... for the
1417         # 'data' field we need to append to our array
1418         for k in marshalled.keys():
1419             if k == 'data':
1420                 self.stream_contents.append(marshalled['data'])
1421             else:
1422                 self.stream_file[k] = marshalled[k]
1423
1424         self.stream_have_file_info = True
1425
1426     # Stream directly from "p4 files" into "git fast-import"
1427     def streamP4Files(self, files):
1428         filesForCommit = []
1429         filesToRead = []
1430         filesToDelete = []
1431
1432         for f in files:
1433             includeFile = True
1434             for val in self.clientSpecDirs:
1435                 if f['path'].startswith(val[0]):
1436                     if val[1][0] <= 0:
1437                         includeFile = False
1438                     break
1439
1440             if includeFile:
1441                 filesForCommit.append(f)
1442                 if f['action'] in self.delete_actions:
1443                     filesToDelete.append(f)
1444                 else:
1445                     filesToRead.append(f)
1446
1447         # deleted files...
1448         for f in filesToDelete:
1449             self.streamOneP4Deletion(f)
1450
1451         if len(filesToRead) > 0:
1452             self.stream_file = {}
1453             self.stream_contents = []
1454             self.stream_have_file_info = False
1455
1456             # curry self argument
1457             def streamP4FilesCbSelf(entry):
1458                 self.streamP4FilesCb(entry)
1459
1460             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1461
1462             p4CmdList(["-x", "-", "print"],
1463                       stdin=fileArgs,
1464                       cb=streamP4FilesCbSelf)
1465
1466             # do the last chunk
1467             if self.stream_file.has_key('depotFile'):
1468                 self.streamOneP4File(self.stream_file, self.stream_contents)
1469
1470     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1471         epoch = details["time"]
1472         author = details["user"]
1473         self.branchPrefixes = branchPrefixes
1474
1475         if self.verbose:
1476             print "commit into %s" % branch
1477
1478         # start with reading files; if that fails, we should not
1479         # create a commit.
1480         new_files = []
1481         for f in files:
1482             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1483                 new_files.append (f)
1484             else:
1485                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1486
1487         self.gitStream.write("commit %s\n" % branch)
1488 #        gitStream.write("mark :%s\n" % details["change"])
1489         self.committedChanges.add(int(details["change"]))
1490         committer = ""
1491         if author not in self.users:
1492             self.getUserMapFromPerforceServer()
1493         if author in self.users:
1494             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1495         else:
1496             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1497
1498         self.gitStream.write("committer %s\n" % committer)
1499
1500         self.gitStream.write("data <<EOT\n")
1501         self.gitStream.write(details["desc"])
1502         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1503                              % (','.join (branchPrefixes), details["change"]))
1504         if len(details['options']) > 0:
1505             self.gitStream.write(": options = %s" % details['options'])
1506         self.gitStream.write("]\nEOT\n\n")
1507
1508         if len(parent) > 0:
1509             if self.verbose:
1510                 print "parent %s" % parent
1511             self.gitStream.write("from %s\n" % parent)
1512
1513         self.streamP4Files(new_files)
1514         self.gitStream.write("\n")
1515
1516         change = int(details["change"])
1517
1518         if self.labels.has_key(change):
1519             label = self.labels[change]
1520             labelDetails = label[0]
1521             labelRevisions = label[1]
1522             if self.verbose:
1523                 print "Change %s is labelled %s" % (change, labelDetails)
1524
1525             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1526                                                     for p in branchPrefixes])
1527
1528             if len(files) == len(labelRevisions):
1529
1530                 cleanedFiles = {}
1531                 for info in files:
1532                     if info["action"] in self.delete_actions:
1533                         continue
1534                     cleanedFiles[info["depotFile"]] = info["rev"]
1535
1536                 if cleanedFiles == labelRevisions:
1537                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1538                     self.gitStream.write("from %s\n" % branch)
1539
1540                     owner = labelDetails["Owner"]
1541                     tagger = ""
1542                     if author in self.users:
1543                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1544                     else:
1545                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1546                     self.gitStream.write("tagger %s\n" % tagger)
1547                     self.gitStream.write("data <<EOT\n")
1548                     self.gitStream.write(labelDetails["Description"])
1549                     self.gitStream.write("EOT\n\n")
1550
1551                 else:
1552                     if not self.silent:
1553                         print ("Tag %s does not match with change %s: files do not match."
1554                                % (labelDetails["label"], change))
1555
1556             else:
1557                 if not self.silent:
1558                     print ("Tag %s does not match with change %s: file count is different."
1559                            % (labelDetails["label"], change))
1560
1561     def getLabels(self):
1562         self.labels = {}
1563
1564         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1565         if len(l) > 0 and not self.silent:
1566             print "Finding files belonging to labels in %s" % `self.depotPaths`
1567
1568         for output in l:
1569             label = output["label"]
1570             revisions = {}
1571             newestChange = 0
1572             if self.verbose:
1573                 print "Querying files for label %s" % label
1574             for file in p4CmdList(["files"] +
1575                                       ["%s...@%s" % (p, label)
1576                                           for p in self.depotPaths]):
1577                 revisions[file["depotFile"]] = file["rev"]
1578                 change = int(file["change"])
1579                 if change > newestChange:
1580                     newestChange = change
1581
1582             self.labels[newestChange] = [output, revisions]
1583
1584         if self.verbose:
1585             print "Label changes: %s" % self.labels.keys()
1586
1587     def guessProjectName(self):
1588         for p in self.depotPaths:
1589             if p.endswith("/"):
1590                 p = p[:-1]
1591             p = p[p.strip().rfind("/") + 1:]
1592             if not p.endswith("/"):
1593                p += "/"
1594             return p
1595
1596     def getBranchMapping(self):
1597         lostAndFoundBranches = set()
1598
1599         user = gitConfig("git-p4.branchUser")
1600         if len(user) > 0:
1601             command = "branches -u %s" % user
1602         else:
1603             command = "branches"
1604
1605         for info in p4CmdList(command):
1606             details = p4Cmd("branch -o %s" % info["branch"])
1607             viewIdx = 0
1608             while details.has_key("View%s" % viewIdx):
1609                 paths = details["View%s" % viewIdx].split(" ")
1610                 viewIdx = viewIdx + 1
1611                 # require standard //depot/foo/... //depot/bar/... mapping
1612                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1613                     continue
1614                 source = paths[0]
1615                 destination = paths[1]
1616                 ## HACK
1617                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1618                     source = source[len(self.depotPaths[0]):-4]
1619                     destination = destination[len(self.depotPaths[0]):-4]
1620
1621                     if destination in self.knownBranches:
1622                         if not self.silent:
1623                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1624                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1625                         continue
1626
1627                     self.knownBranches[destination] = source
1628
1629                     lostAndFoundBranches.discard(destination)
1630
1631                     if source not in self.knownBranches:
1632                         lostAndFoundBranches.add(source)
1633
1634         # Perforce does not strictly require branches to be defined, so we also
1635         # check git config for a branch list.
1636         #
1637         # Example of branch definition in git config file:
1638         # [git-p4]
1639         #   branchList=main:branchA
1640         #   branchList=main:branchB
1641         #   branchList=branchA:branchC
1642         configBranches = gitConfigList("git-p4.branchList")
1643         for branch in configBranches:
1644             if branch:
1645                 (source, destination) = branch.split(":")
1646                 self.knownBranches[destination] = source
1647
1648                 lostAndFoundBranches.discard(destination)
1649
1650                 if source not in self.knownBranches:
1651                     lostAndFoundBranches.add(source)
1652
1653
1654         for branch in lostAndFoundBranches:
1655             self.knownBranches[branch] = branch
1656
1657     def getBranchMappingFromGitBranches(self):
1658         branches = p4BranchesInGit(self.importIntoRemotes)
1659         for branch in branches.keys():
1660             if branch == "master":
1661                 branch = "main"
1662             else:
1663                 branch = branch[len(self.projectName):]
1664             self.knownBranches[branch] = branch
1665
1666     def listExistingP4GitBranches(self):
1667         # branches holds mapping from name to commit
1668         branches = p4BranchesInGit(self.importIntoRemotes)
1669         self.p4BranchesInGit = branches.keys()
1670         for branch in branches.keys():
1671             self.initialParents[self.refPrefix + branch] = branches[branch]
1672
1673     def updateOptionDict(self, d):
1674         option_keys = {}
1675         if self.keepRepoPath:
1676             option_keys['keepRepoPath'] = 1
1677
1678         d["options"] = ' '.join(sorted(option_keys.keys()))
1679
1680     def readOptions(self, d):
1681         self.keepRepoPath = (d.has_key('options')
1682                              and ('keepRepoPath' in d['options']))
1683
1684     def gitRefForBranch(self, branch):
1685         if branch == "main":
1686             return self.refPrefix + "master"
1687
1688         if len(branch) <= 0:
1689             return branch
1690
1691         return self.refPrefix + self.projectName + branch
1692
1693     def gitCommitByP4Change(self, ref, change):
1694         if self.verbose:
1695             print "looking in ref " + ref + " for change %s using bisect..." % change
1696
1697         earliestCommit = ""
1698         latestCommit = parseRevision(ref)
1699
1700         while True:
1701             if self.verbose:
1702                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1703             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1704             if len(next) == 0:
1705                 if self.verbose:
1706                     print "argh"
1707                 return ""
1708             log = extractLogMessageFromGitCommit(next)
1709             settings = extractSettingsGitLog(log)
1710             currentChange = int(settings['change'])
1711             if self.verbose:
1712                 print "current change %s" % currentChange
1713
1714             if currentChange == change:
1715                 if self.verbose:
1716                     print "found %s" % next
1717                 return next
1718
1719             if currentChange < change:
1720                 earliestCommit = "^%s" % next
1721             else:
1722                 latestCommit = "%s" % next
1723
1724         return ""
1725
1726     def importNewBranch(self, branch, maxChange):
1727         # make fast-import flush all changes to disk and update the refs using the checkpoint
1728         # command so that we can try to find the branch parent in the git history
1729         self.gitStream.write("checkpoint\n\n");
1730         self.gitStream.flush();
1731         branchPrefix = self.depotPaths[0] + branch + "/"
1732         range = "@1,%s" % maxChange
1733         #print "prefix" + branchPrefix
1734         changes = p4ChangesForPaths([branchPrefix], range)
1735         if len(changes) <= 0:
1736             return False
1737         firstChange = changes[0]
1738         #print "first change in branch: %s" % firstChange
1739         sourceBranch = self.knownBranches[branch]
1740         sourceDepotPath = self.depotPaths[0] + sourceBranch
1741         sourceRef = self.gitRefForBranch(sourceBranch)
1742         #print "source " + sourceBranch
1743
1744         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1745         #print "branch parent: %s" % branchParentChange
1746         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1747         if len(gitParent) > 0:
1748             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1749             #print "parent git commit: %s" % gitParent
1750
1751         self.importChanges(changes)
1752         return True
1753
1754     def importChanges(self, changes):
1755         cnt = 1
1756         for change in changes:
1757             description = p4Cmd("describe %s" % change)
1758             self.updateOptionDict(description)
1759
1760             if not self.silent:
1761                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1762                 sys.stdout.flush()
1763             cnt = cnt + 1
1764
1765             try:
1766                 if self.detectBranches:
1767                     branches = self.splitFilesIntoBranches(description)
1768                     for branch in branches.keys():
1769                         ## HACK  --hwn
1770                         branchPrefix = self.depotPaths[0] + branch + "/"
1771
1772                         parent = ""
1773
1774                         filesForCommit = branches[branch]
1775
1776                         if self.verbose:
1777                             print "branch is %s" % branch
1778
1779                         self.updatedBranches.add(branch)
1780
1781                         if branch not in self.createdBranches:
1782                             self.createdBranches.add(branch)
1783                             parent = self.knownBranches[branch]
1784                             if parent == branch:
1785                                 parent = ""
1786                             else:
1787                                 fullBranch = self.projectName + branch
1788                                 if fullBranch not in self.p4BranchesInGit:
1789                                     if not self.silent:
1790                                         print("\n    Importing new branch %s" % fullBranch);
1791                                     if self.importNewBranch(branch, change - 1):
1792                                         parent = ""
1793                                         self.p4BranchesInGit.append(fullBranch)
1794                                     if not self.silent:
1795                                         print("\n    Resuming with change %s" % change);
1796
1797                                 if self.verbose:
1798                                     print "parent determined through known branches: %s" % parent
1799
1800                         branch = self.gitRefForBranch(branch)
1801                         parent = self.gitRefForBranch(parent)
1802
1803                         if self.verbose:
1804                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1805
1806                         if len(parent) == 0 and branch in self.initialParents:
1807                             parent = self.initialParents[branch]
1808                             del self.initialParents[branch]
1809
1810                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1811                 else:
1812                     files = self.extractFilesFromCommit(description)
1813                     self.commit(description, files, self.branch, self.depotPaths,
1814                                 self.initialParent)
1815                     self.initialParent = ""
1816             except IOError:
1817                 print self.gitError.read()
1818                 sys.exit(1)
1819
1820     def importHeadRevision(self, revision):
1821         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1822
1823         details = {}
1824         details["user"] = "git perforce import user"
1825         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1826                            % (' '.join(self.depotPaths), revision))
1827         details["change"] = revision
1828         newestRevision = 0
1829
1830         fileCnt = 0
1831         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
1832
1833         for info in p4CmdList(["files"] + fileArgs):
1834
1835             if 'code' in info and info['code'] == 'error':
1836                 sys.stderr.write("p4 returned an error: %s\n"
1837                                  % info['data'])
1838                 if info['data'].find("must refer to client") >= 0:
1839                     sys.stderr.write("This particular p4 error is misleading.\n")
1840                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1841                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1842                 sys.exit(1)
1843             if 'p4ExitCode' in info:
1844                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1845                 sys.exit(1)
1846
1847
1848             change = int(info["change"])
1849             if change > newestRevision:
1850                 newestRevision = change
1851
1852             if info["action"] in self.delete_actions:
1853                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1854                 #fileCnt = fileCnt + 1
1855                 continue
1856
1857             for prop in ["depotFile", "rev", "action", "type" ]:
1858                 details["%s%s" % (prop, fileCnt)] = info[prop]
1859
1860             fileCnt = fileCnt + 1
1861
1862         details["change"] = newestRevision
1863
1864         # Use time from top-most change so that all git-p4 clones of
1865         # the same p4 repo have the same commit SHA1s.
1866         res = p4CmdList("describe -s %d" % newestRevision)
1867         newestTime = None
1868         for r in res:
1869             if r.has_key('time'):
1870                 newestTime = int(r['time'])
1871         if newestTime is None:
1872             die("\"describe -s\" on newest change %d did not give a time")
1873         details["time"] = newestTime
1874
1875         self.updateOptionDict(details)
1876         try:
1877             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1878         except IOError:
1879             print "IO error with git fast-import. Is your git version recent enough?"
1880             print self.gitError.read()
1881
1882
1883     def getClientSpec(self):
1884         specList = p4CmdList( "client -o" )
1885         temp = {}
1886         for entry in specList:
1887             for k,v in entry.iteritems():
1888                 if k.startswith("View"):
1889
1890                     # p4 has these %%1 to %%9 arguments in specs to
1891                     # reorder paths; which we can't handle (yet :)
1892                     if re.search('%%\d', v) != None:
1893                         print "Sorry, can't handle %%n arguments in client specs"
1894                         sys.exit(1)
1895                     if re.search('\*', v) != None:
1896                         print "Sorry, can't handle * mappings in client specs"
1897                         sys.exit(1)
1898
1899                     if v.startswith('"'):
1900                         start = 1
1901                     else:
1902                         start = 0
1903                     index = v.find("...")
1904
1905                     # save the "client view"; i.e the RHS of the view
1906                     # line that tells the client where to put the
1907                     # files for this view.
1908                     cv = v[index+3:].strip() # +3 to remove previous '...'
1909
1910                     # if the client view doesn't end with a
1911                     # ... wildcard, then we're going to mess up the
1912                     # output directory, so fail gracefully.
1913                     if not cv.endswith('...'):
1914                         print 'Sorry, client view in "%s" needs to end with wildcard' % (k)
1915                         sys.exit(1)
1916                     cv=cv[:-3]
1917
1918                     # now save the view; +index means included, -index
1919                     # means it should be filtered out.
1920                     v = v[start:index]
1921                     if v.startswith("-"):
1922                         v = v[1:]
1923                         include = -len(v)
1924                     else:
1925                         include = len(v)
1926
1927                     # store the View #number for sorting
1928                     # and the View string itself (this last for documentation)
1929                     temp[v] = (include, cv, int(k[4:]),k)
1930
1931         self.clientSpecDirs = temp.items()
1932         # Perforce ViewNN with higher #numbers override those with lower
1933         # reverse sort on the View #number
1934         self.clientSpecDirs.sort( lambda x, y:  y[1][2] -  x[1][2]  )
1935         if self.verbose:
1936             for val in self.clientSpecDirs:
1937                     print "clientSpecDirs: %s %s" % (val[0],val[1])
1938
1939     def run(self, args):
1940         self.depotPaths = []
1941         self.changeRange = ""
1942         self.initialParent = ""
1943         self.previousDepotPaths = []
1944
1945         # map from branch depot path to parent branch
1946         self.knownBranches = {}
1947         self.initialParents = {}
1948         self.hasOrigin = originP4BranchesExist()
1949         if not self.syncWithOrigin:
1950             self.hasOrigin = False
1951
1952         if self.importIntoRemotes:
1953             self.refPrefix = "refs/remotes/p4/"
1954         else:
1955             self.refPrefix = "refs/heads/p4/"
1956
1957         if self.syncWithOrigin and self.hasOrigin:
1958             if not self.silent:
1959                 print "Syncing with origin first by calling git fetch origin"
1960             system("git fetch origin")
1961
1962         if len(self.branch) == 0:
1963             self.branch = self.refPrefix + "master"
1964             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1965                 system("git update-ref %s refs/heads/p4" % self.branch)
1966                 system("git branch -D p4");
1967             # create it /after/ importing, when master exists
1968             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1969                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1970
1971         if not self.useClientSpec:
1972             if gitConfig("git-p4.useclientspec", "--bool") == "true":
1973                 self.useClientSpec = True
1974         if self.useClientSpec:
1975             self.getClientSpec()
1976
1977         # TODO: should always look at previous commits,
1978         # merge with previous imports, if possible.
1979         if args == []:
1980             if self.hasOrigin:
1981                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1982             self.listExistingP4GitBranches()
1983
1984             if len(self.p4BranchesInGit) > 1:
1985                 if not self.silent:
1986                     print "Importing from/into multiple branches"
1987                 self.detectBranches = True
1988
1989             if self.verbose:
1990                 print "branches: %s" % self.p4BranchesInGit
1991
1992             p4Change = 0
1993             for branch in self.p4BranchesInGit:
1994                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
1995
1996                 settings = extractSettingsGitLog(logMsg)
1997
1998                 self.readOptions(settings)
1999                 if (settings.has_key('depot-paths')
2000                     and settings.has_key ('change')):
2001                     change = int(settings['change']) + 1
2002                     p4Change = max(p4Change, change)
2003
2004                     depotPaths = sorted(settings['depot-paths'])
2005                     if self.previousDepotPaths == []:
2006                         self.previousDepotPaths = depotPaths
2007                     else:
2008                         paths = []
2009                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2010                             prev_list = prev.split("/")
2011                             cur_list = cur.split("/")
2012                             for i in range(0, min(len(cur_list), len(prev_list))):
2013                                 if cur_list[i] <> prev_list[i]:
2014                                     i = i - 1
2015                                     break
2016
2017                             paths.append ("/".join(cur_list[:i + 1]))
2018
2019                         self.previousDepotPaths = paths
2020
2021             if p4Change > 0:
2022                 self.depotPaths = sorted(self.previousDepotPaths)
2023                 self.changeRange = "@%s,#head" % p4Change
2024                 if not self.detectBranches:
2025                     self.initialParent = parseRevision(self.branch)
2026                 if not self.silent and not self.detectBranches:
2027                     print "Performing incremental import into %s git branch" % self.branch
2028
2029         if not self.branch.startswith("refs/"):
2030             self.branch = "refs/heads/" + self.branch
2031
2032         if len(args) == 0 and self.depotPaths:
2033             if not self.silent:
2034                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2035         else:
2036             if self.depotPaths and self.depotPaths != args:
2037                 print ("previous import used depot path %s and now %s was specified. "
2038                        "This doesn't work!" % (' '.join (self.depotPaths),
2039                                                ' '.join (args)))
2040                 sys.exit(1)
2041
2042             self.depotPaths = sorted(args)
2043
2044         revision = ""
2045         self.users = {}
2046
2047         # Make sure no revision specifiers are used when --changesfile
2048         # is specified.
2049         bad_changesfile = False
2050         if len(self.changesFile) > 0:
2051             for p in self.depotPaths:
2052                 if p.find("@") >= 0 or p.find("#") >= 0:
2053                     bad_changesfile = True
2054                     break
2055         if bad_changesfile:
2056             die("Option --changesfile is incompatible with revision specifiers")
2057
2058         newPaths = []
2059         for p in self.depotPaths:
2060             if p.find("@") != -1:
2061                 atIdx = p.index("@")
2062                 self.changeRange = p[atIdx:]
2063                 if self.changeRange == "@all":
2064                     self.changeRange = ""
2065                 elif ',' not in self.changeRange:
2066                     revision = self.changeRange
2067                     self.changeRange = ""
2068                 p = p[:atIdx]
2069             elif p.find("#") != -1:
2070                 hashIdx = p.index("#")
2071                 revision = p[hashIdx:]
2072                 p = p[:hashIdx]
2073             elif self.previousDepotPaths == []:
2074                 # pay attention to changesfile, if given, else import
2075                 # the entire p4 tree at the head revision
2076                 if len(self.changesFile) == 0:
2077                     revision = "#head"
2078
2079             p = re.sub ("\.\.\.$", "", p)
2080             if not p.endswith("/"):
2081                 p += "/"
2082
2083             newPaths.append(p)
2084
2085         self.depotPaths = newPaths
2086
2087
2088         self.loadUserMapFromCache()
2089         self.labels = {}
2090         if self.detectLabels:
2091             self.getLabels();
2092
2093         if self.detectBranches:
2094             ## FIXME - what's a P4 projectName ?
2095             self.projectName = self.guessProjectName()
2096
2097             if self.hasOrigin:
2098                 self.getBranchMappingFromGitBranches()
2099             else:
2100                 self.getBranchMapping()
2101             if self.verbose:
2102                 print "p4-git branches: %s" % self.p4BranchesInGit
2103                 print "initial parents: %s" % self.initialParents
2104             for b in self.p4BranchesInGit:
2105                 if b != "master":
2106
2107                     ## FIXME
2108                     b = b[len(self.projectName):]
2109                 self.createdBranches.add(b)
2110
2111         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2112
2113         importProcess = subprocess.Popen(["git", "fast-import"],
2114                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2115                                          stderr=subprocess.PIPE);
2116         self.gitOutput = importProcess.stdout
2117         self.gitStream = importProcess.stdin
2118         self.gitError = importProcess.stderr
2119
2120         if revision:
2121             self.importHeadRevision(revision)
2122         else:
2123             changes = []
2124
2125             if len(self.changesFile) > 0:
2126                 output = open(self.changesFile).readlines()
2127                 changeSet = set()
2128                 for line in output:
2129                     changeSet.add(int(line))
2130
2131                 for change in changeSet:
2132                     changes.append(change)
2133
2134                 changes.sort()
2135             else:
2136                 # catch "git-p4 sync" with no new branches, in a repo that
2137                 # does not have any existing git-p4 branches
2138                 if len(args) == 0 and not self.p4BranchesInGit:
2139                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2140                 if self.verbose:
2141                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2142                                                               self.changeRange)
2143                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2144
2145                 if len(self.maxChanges) > 0:
2146                     changes = changes[:min(int(self.maxChanges), len(changes))]
2147
2148             if len(changes) == 0:
2149                 if not self.silent:
2150                     print "No changes to import!"
2151                 return True
2152
2153             if not self.silent and not self.detectBranches:
2154                 print "Import destination: %s" % self.branch
2155
2156             self.updatedBranches = set()
2157
2158             self.importChanges(changes)
2159
2160             if not self.silent:
2161                 print ""
2162                 if len(self.updatedBranches) > 0:
2163                     sys.stdout.write("Updated branches: ")
2164                     for b in self.updatedBranches:
2165                         sys.stdout.write("%s " % b)
2166                     sys.stdout.write("\n")
2167
2168         self.gitStream.close()
2169         if importProcess.wait() != 0:
2170             die("fast-import failed: %s" % self.gitError.read())
2171         self.gitOutput.close()
2172         self.gitError.close()
2173
2174         return True
2175
2176 class P4Rebase(Command):
2177     def __init__(self):
2178         Command.__init__(self)
2179         self.options = [ ]
2180         self.description = ("Fetches the latest revision from perforce and "
2181                             + "rebases the current work (branch) against it")
2182         self.verbose = False
2183
2184     def run(self, args):
2185         sync = P4Sync()
2186         sync.run([])
2187
2188         return self.rebase()
2189
2190     def rebase(self):
2191         if os.system("git update-index --refresh") != 0:
2192             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2193         if len(read_pipe("git diff-index HEAD --")) > 0:
2194             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2195
2196         [upstream, settings] = findUpstreamBranchPoint()
2197         if len(upstream) == 0:
2198             die("Cannot find upstream branchpoint for rebase")
2199
2200         # the branchpoint may be p4/foo~3, so strip off the parent
2201         upstream = re.sub("~[0-9]+$", "", upstream)
2202
2203         print "Rebasing the current branch onto %s" % upstream
2204         oldHead = read_pipe("git rev-parse HEAD").strip()
2205         system("git rebase %s" % upstream)
2206         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2207         return True
2208
2209 class P4Clone(P4Sync):
2210     def __init__(self):
2211         P4Sync.__init__(self)
2212         self.description = "Creates a new git repository and imports from Perforce into it"
2213         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2214         self.options += [
2215             optparse.make_option("--destination", dest="cloneDestination",
2216                                  action='store', default=None,
2217                                  help="where to leave result of the clone"),
2218             optparse.make_option("-/", dest="cloneExclude",
2219                                  action="append", type="string",
2220                                  help="exclude depot path"),
2221             optparse.make_option("--bare", dest="cloneBare",
2222                                  action="store_true", default=False),
2223         ]
2224         self.cloneDestination = None
2225         self.needsGit = False
2226         self.cloneBare = False
2227
2228     # This is required for the "append" cloneExclude action
2229     def ensure_value(self, attr, value):
2230         if not hasattr(self, attr) or getattr(self, attr) is None:
2231             setattr(self, attr, value)
2232         return getattr(self, attr)
2233
2234     def defaultDestination(self, args):
2235         ## TODO: use common prefix of args?
2236         depotPath = args[0]
2237         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2238         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2239         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2240         depotDir = re.sub(r"/$", "", depotDir)
2241         return os.path.split(depotDir)[1]
2242
2243     def run(self, args):
2244         if len(args) < 1:
2245             return False
2246
2247         if self.keepRepoPath and not self.cloneDestination:
2248             sys.stderr.write("Must specify destination for --keep-path\n")
2249             sys.exit(1)
2250
2251         depotPaths = args
2252
2253         if not self.cloneDestination and len(depotPaths) > 1:
2254             self.cloneDestination = depotPaths[-1]
2255             depotPaths = depotPaths[:-1]
2256
2257         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2258         for p in depotPaths:
2259             if not p.startswith("//"):
2260                 return False
2261
2262         if not self.cloneDestination:
2263             self.cloneDestination = self.defaultDestination(args)
2264
2265         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2266
2267         if not os.path.exists(self.cloneDestination):
2268             os.makedirs(self.cloneDestination)
2269         chdir(self.cloneDestination)
2270
2271         init_cmd = [ "git", "init" ]
2272         if self.cloneBare:
2273             init_cmd.append("--bare")
2274         subprocess.check_call(init_cmd)
2275
2276         if not P4Sync.run(self, depotPaths):
2277             return False
2278         if self.branch != "master":
2279             if self.importIntoRemotes:
2280                 masterbranch = "refs/remotes/p4/master"
2281             else:
2282                 masterbranch = "refs/heads/p4/master"
2283             if gitBranchExists(masterbranch):
2284                 system("git branch master %s" % masterbranch)
2285                 if not self.cloneBare:
2286                     system("git checkout -f")
2287             else:
2288                 print "Could not detect main branch. No checkout/master branch created."
2289
2290         return True
2291
2292 class P4Branches(Command):
2293     def __init__(self):
2294         Command.__init__(self)
2295         self.options = [ ]
2296         self.description = ("Shows the git branches that hold imports and their "
2297                             + "corresponding perforce depot paths")
2298         self.verbose = False
2299
2300     def run(self, args):
2301         if originP4BranchesExist():
2302             createOrUpdateBranchesFromOrigin()
2303
2304         cmdline = "git rev-parse --symbolic "
2305         cmdline += " --remotes"
2306
2307         for line in read_pipe_lines(cmdline):
2308             line = line.strip()
2309
2310             if not line.startswith('p4/') or line == "p4/HEAD":
2311                 continue
2312             branch = line
2313
2314             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2315             settings = extractSettingsGitLog(log)
2316
2317             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2318         return True
2319
2320 class HelpFormatter(optparse.IndentedHelpFormatter):
2321     def __init__(self):
2322         optparse.IndentedHelpFormatter.__init__(self)
2323
2324     def format_description(self, description):
2325         if description:
2326             return description + "\n"
2327         else:
2328             return ""
2329
2330 def printUsage(commands):
2331     print "usage: %s <command> [options]" % sys.argv[0]
2332     print ""
2333     print "valid commands: %s" % ", ".join(commands)
2334     print ""
2335     print "Try %s <command> --help for command specific help." % sys.argv[0]
2336     print ""
2337
2338 commands = {
2339     "debug" : P4Debug,
2340     "submit" : P4Submit,
2341     "commit" : P4Submit,
2342     "sync" : P4Sync,
2343     "rebase" : P4Rebase,
2344     "clone" : P4Clone,
2345     "rollback" : P4RollBack,
2346     "branches" : P4Branches
2347 }
2348
2349
2350 def main():
2351     if len(sys.argv[1:]) == 0:
2352         printUsage(commands.keys())
2353         sys.exit(2)
2354
2355     cmd = ""
2356     cmdName = sys.argv[1]
2357     try:
2358         klass = commands[cmdName]
2359         cmd = klass()
2360     except KeyError:
2361         print "unknown command %s" % cmdName
2362         print ""
2363         printUsage(commands.keys())
2364         sys.exit(2)
2365
2366     options = cmd.options
2367     cmd.gitdir = os.environ.get("GIT_DIR", None)
2368
2369     args = sys.argv[2:]
2370
2371     if len(options) > 0:
2372         if cmd.needsGit:
2373             options.append(optparse.make_option("--git-dir", dest="gitdir"))
2374
2375         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2376                                        options,
2377                                        description = cmd.description,
2378                                        formatter = HelpFormatter())
2379
2380         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2381     global verbose
2382     verbose = cmd.verbose
2383     if cmd.needsGit:
2384         if cmd.gitdir == None:
2385             cmd.gitdir = os.path.abspath(".git")
2386             if not isValidGitDir(cmd.gitdir):
2387                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2388                 if os.path.exists(cmd.gitdir):
2389                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2390                     if len(cdup) > 0:
2391                         chdir(cdup);
2392
2393         if not isValidGitDir(cmd.gitdir):
2394             if isValidGitDir(cmd.gitdir + "/.git"):
2395                 cmd.gitdir += "/.git"
2396             else:
2397                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2398
2399         os.environ["GIT_DIR"] = cmd.gitdir
2400
2401     if not cmd.run(args):
2402         parser.print_help()
2403         sys.exit(2)
2404
2405
2406 if __name__ == '__main__':
2407     main()