Code

8f28e0a6fbe8924a78ccb5bc180d445b464da354
[git.git] / contrib / fast-import / git-p4
1 #!/usr/bin/env python
2 #
3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
4 #
5 # Author: Simon Hausmann <simon@lst.de>
6 # Copyright: 2007 Simon Hausmann <simon@lst.de>
7 #            2007 Trolltech ASA
8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
9 #
11 import optparse, sys, os, marshal, subprocess, shelve
12 import tempfile, getopt, os.path, time, platform
13 import re
15 verbose = False
18 def p4_build_cmd(cmd):
19     """Build a suitable p4 command line.
21     This consolidates building and returning a p4 command line into one
22     location. It means that hooking into the environment, or other configuration
23     can be done more easily.
24     """
25     real_cmd = ["p4"]
27     user = gitConfig("git-p4.user")
28     if len(user) > 0:
29         real_cmd += ["-u",user]
31     password = gitConfig("git-p4.password")
32     if len(password) > 0:
33         real_cmd += ["-P", password]
35     port = gitConfig("git-p4.port")
36     if len(port) > 0:
37         real_cmd += ["-p", port]
39     host = gitConfig("git-p4.host")
40     if len(host) > 0:
41         real_cmd += ["-h", host]
43     client = gitConfig("git-p4.client")
44     if len(client) > 0:
45         real_cmd += ["-c", client]
48     if isinstance(cmd,basestring):
49         real_cmd = ' '.join(real_cmd) + ' ' + cmd
50     else:
51         real_cmd += cmd
52     return real_cmd
54 def chdir(dir):
55     # P4 uses the PWD environment variable rather than getcwd(). Since we're
56     # not using the shell, we have to set it ourselves.  This path could
57     # be relative, so go there first, then figure out where we ended up.
58     os.chdir(dir)
59     os.environ['PWD'] = os.getcwd()
61 def die(msg):
62     if verbose:
63         raise Exception(msg)
64     else:
65         sys.stderr.write(msg + "\n")
66         sys.exit(1)
68 def write_pipe(c, stdin):
69     if verbose:
70         sys.stderr.write('Writing pipe: %s\n' % str(c))
72     expand = isinstance(c,basestring)
73     p = subprocess.Popen(c, stdin=subprocess.PIPE, shell=expand)
74     pipe = p.stdin
75     val = pipe.write(stdin)
76     pipe.close()
77     if p.wait():
78         die('Command failed: %s' % str(c))
80     return val
82 def p4_write_pipe(c, stdin):
83     real_cmd = p4_build_cmd(c)
84     return write_pipe(real_cmd, stdin)
86 def read_pipe(c, ignore_error=False):
87     if verbose:
88         sys.stderr.write('Reading pipe: %s\n' % str(c))
90     expand = isinstance(c,basestring)
91     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
92     pipe = p.stdout
93     val = pipe.read()
94     if p.wait() and not ignore_error:
95         die('Command failed: %s' % str(c))
97     return val
99 def p4_read_pipe(c, ignore_error=False):
100     real_cmd = p4_build_cmd(c)
101     return read_pipe(real_cmd, ignore_error)
103 def read_pipe_lines(c):
104     if verbose:
105         sys.stderr.write('Reading pipe: %s\n' % str(c))
107     expand = isinstance(c, basestring)
108     p = subprocess.Popen(c, stdout=subprocess.PIPE, shell=expand)
109     pipe = p.stdout
110     val = pipe.readlines()
111     if pipe.close() or p.wait():
112         die('Command failed: %s' % str(c))
114     return val
116 def p4_read_pipe_lines(c):
117     """Specifically invoke p4 on the command supplied. """
118     real_cmd = p4_build_cmd(c)
119     return read_pipe_lines(real_cmd)
121 def system(cmd):
122     expand = isinstance(cmd,basestring)
123     if verbose:
124         sys.stderr.write("executing %s\n" % str(cmd))
125     subprocess.check_call(cmd, shell=expand)
127 def p4_system(cmd):
128     """Specifically invoke p4 as the system command. """
129     real_cmd = p4_build_cmd(cmd)
130     expand = isinstance(real_cmd, basestring)
131     subprocess.check_call(real_cmd, shell=expand)
133 def p4_integrate(src, dest):
134     p4_system(["integrate", "-Dt", src, dest])
136 def p4_sync(path):
137     p4_system(["sync", path])
139 def p4_add(f):
140     p4_system(["add", f])
142 def p4_delete(f):
143     p4_system(["delete", f])
145 def p4_edit(f):
146     p4_system(["edit", f])
148 def p4_revert(f):
149     p4_system(["revert", f])
151 def p4_reopen(type, file):
152     p4_system(["reopen", "-t", type, file])
155 # Canonicalize the p4 type and return a tuple of the
156 # base type, plus any modifiers.  See "p4 help filetypes"
157 # for a list and explanation.
159 def split_p4_type(p4type):
161     p4_filetypes_historical = {
162         "ctempobj": "binary+Sw",
163         "ctext": "text+C",
164         "cxtext": "text+Cx",
165         "ktext": "text+k",
166         "kxtext": "text+kx",
167         "ltext": "text+F",
168         "tempobj": "binary+FSw",
169         "ubinary": "binary+F",
170         "uresource": "resource+F",
171         "uxbinary": "binary+Fx",
172         "xbinary": "binary+x",
173         "xltext": "text+Fx",
174         "xtempobj": "binary+Swx",
175         "xtext": "text+x",
176         "xunicode": "unicode+x",
177         "xutf16": "utf16+x",
178     }
179     if p4type in p4_filetypes_historical:
180         p4type = p4_filetypes_historical[p4type]
181     mods = ""
182     s = p4type.split("+")
183     base = s[0]
184     mods = ""
185     if len(s) > 1:
186         mods = s[1]
187     return (base, mods)
190 def setP4ExecBit(file, mode):
191     # Reopens an already open file and changes the execute bit to match
192     # the execute bit setting in the passed in mode.
194     p4Type = "+x"
196     if not isModeExec(mode):
197         p4Type = getP4OpenedType(file)
198         p4Type = re.sub('^([cku]?)x(.*)', '\\1\\2', p4Type)
199         p4Type = re.sub('(.*?\+.*?)x(.*?)', '\\1\\2', p4Type)
200         if p4Type[-1] == "+":
201             p4Type = p4Type[0:-1]
203     p4_reopen(p4Type, file)
205 def getP4OpenedType(file):
206     # Returns the perforce file type for the given file.
208     result = p4_read_pipe(["opened", file])
209     match = re.match(".*\((.+)\)\r?$", result)
210     if match:
211         return match.group(1)
212     else:
213         die("Could not determine file type for %s (result: '%s')" % (file, result))
215 def diffTreePattern():
216     # This is a simple generator for the diff tree regex pattern. This could be
217     # a class variable if this and parseDiffTreeEntry were a part of a class.
218     pattern = re.compile(':(\d+) (\d+) (\w+) (\w+) ([A-Z])(\d+)?\t(.*?)((\t(.*))|$)')
219     while True:
220         yield pattern
222 def parseDiffTreeEntry(entry):
223     """Parses a single diff tree entry into its component elements.
225     See git-diff-tree(1) manpage for details about the format of the diff
226     output. This method returns a dictionary with the following elements:
228     src_mode - The mode of the source file
229     dst_mode - The mode of the destination file
230     src_sha1 - The sha1 for the source file
231     dst_sha1 - The sha1 fr the destination file
232     status - The one letter status of the diff (i.e. 'A', 'M', 'D', etc)
233     status_score - The score for the status (applicable for 'C' and 'R'
234                    statuses). This is None if there is no score.
235     src - The path for the source file.
236     dst - The path for the destination file. This is only present for
237           copy or renames. If it is not present, this is None.
239     If the pattern is not matched, None is returned."""
241     match = diffTreePattern().next().match(entry)
242     if match:
243         return {
244             'src_mode': match.group(1),
245             'dst_mode': match.group(2),
246             'src_sha1': match.group(3),
247             'dst_sha1': match.group(4),
248             'status': match.group(5),
249             'status_score': match.group(6),
250             'src': match.group(7),
251             'dst': match.group(10)
252         }
253     return None
255 def isModeExec(mode):
256     # Returns True if the given git mode represents an executable file,
257     # otherwise False.
258     return mode[-3:] == "755"
260 def isModeExecChanged(src_mode, dst_mode):
261     return isModeExec(src_mode) != isModeExec(dst_mode)
263 def p4CmdList(cmd, stdin=None, stdin_mode='w+b', cb=None):
265     if isinstance(cmd,basestring):
266         cmd = "-G " + cmd
267         expand = True
268     else:
269         cmd = ["-G"] + cmd
270         expand = False
272     cmd = p4_build_cmd(cmd)
273     if verbose:
274         sys.stderr.write("Opening pipe: %s\n" % str(cmd))
276     # Use a temporary file to avoid deadlocks without
277     # subprocess.communicate(), which would put another copy
278     # of stdout into memory.
279     stdin_file = None
280     if stdin is not None:
281         stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode)
282         if isinstance(stdin,basestring):
283             stdin_file.write(stdin)
284         else:
285             for i in stdin:
286                 stdin_file.write(i + '\n')
287         stdin_file.flush()
288         stdin_file.seek(0)
290     p4 = subprocess.Popen(cmd,
291                           shell=expand,
292                           stdin=stdin_file,
293                           stdout=subprocess.PIPE)
295     result = []
296     try:
297         while True:
298             entry = marshal.load(p4.stdout)
299             if cb is not None:
300                 cb(entry)
301             else:
302                 result.append(entry)
303     except EOFError:
304         pass
305     exitCode = p4.wait()
306     if exitCode != 0:
307         entry = {}
308         entry["p4ExitCode"] = exitCode
309         result.append(entry)
311     return result
313 def p4Cmd(cmd):
314     list = p4CmdList(cmd)
315     result = {}
316     for entry in list:
317         result.update(entry)
318     return result;
320 def p4Where(depotPath):
321     if not depotPath.endswith("/"):
322         depotPath += "/"
323     depotPath = depotPath + "..."
324     outputList = p4CmdList(["where", depotPath])
325     output = None
326     for entry in outputList:
327         if "depotFile" in entry:
328             if entry["depotFile"] == depotPath:
329                 output = entry
330                 break
331         elif "data" in entry:
332             data = entry.get("data")
333             space = data.find(" ")
334             if data[:space] == depotPath:
335                 output = entry
336                 break
337     if output == None:
338         return ""
339     if output["code"] == "error":
340         return ""
341     clientPath = ""
342     if "path" in output:
343         clientPath = output.get("path")
344     elif "data" in output:
345         data = output.get("data")
346         lastSpace = data.rfind(" ")
347         clientPath = data[lastSpace + 1:]
349     if clientPath.endswith("..."):
350         clientPath = clientPath[:-3]
351     return clientPath
353 def currentGitBranch():
354     return read_pipe("git name-rev HEAD").split(" ")[1].strip()
356 def isValidGitDir(path):
357     if (os.path.exists(path + "/HEAD")
358         and os.path.exists(path + "/refs") and os.path.exists(path + "/objects")):
359         return True;
360     return False
362 def parseRevision(ref):
363     return read_pipe("git rev-parse %s" % ref).strip()
365 def branchExists(ref):
366     rev = read_pipe(["git", "rev-parse", "-q", "--verify", ref],
367                      ignore_error=True)
368     return len(rev) > 0
370 def extractLogMessageFromGitCommit(commit):
371     logMessage = ""
373     ## fixme: title is first line of commit, not 1st paragraph.
374     foundTitle = False
375     for log in read_pipe_lines("git cat-file commit %s" % commit):
376        if not foundTitle:
377            if len(log) == 1:
378                foundTitle = True
379            continue
381        logMessage += log
382     return logMessage
384 def extractSettingsGitLog(log):
385     values = {}
386     for line in log.split("\n"):
387         line = line.strip()
388         m = re.search (r"^ *\[git-p4: (.*)\]$", line)
389         if not m:
390             continue
392         assignments = m.group(1).split (':')
393         for a in assignments:
394             vals = a.split ('=')
395             key = vals[0].strip()
396             val = ('='.join (vals[1:])).strip()
397             if val.endswith ('\"') and val.startswith('"'):
398                 val = val[1:-1]
400             values[key] = val
402     paths = values.get("depot-paths")
403     if not paths:
404         paths = values.get("depot-path")
405     if paths:
406         values['depot-paths'] = paths.split(',')
407     return values
409 def gitBranchExists(branch):
410     proc = subprocess.Popen(["git", "rev-parse", branch],
411                             stderr=subprocess.PIPE, stdout=subprocess.PIPE);
412     return proc.wait() == 0;
414 _gitConfig = {}
415 def gitConfig(key, args = None): # set args to "--bool", for instance
416     if not _gitConfig.has_key(key):
417         argsFilter = ""
418         if args != None:
419             argsFilter = "%s " % args
420         cmd = "git config %s%s" % (argsFilter, key)
421         _gitConfig[key] = read_pipe(cmd, ignore_error=True).strip()
422     return _gitConfig[key]
424 def gitConfigList(key):
425     if not _gitConfig.has_key(key):
426         _gitConfig[key] = read_pipe("git config --get-all %s" % key, ignore_error=True).strip().split(os.linesep)
427     return _gitConfig[key]
429 def p4BranchesInGit(branchesAreInRemotes = True):
430     branches = {}
432     cmdline = "git rev-parse --symbolic "
433     if branchesAreInRemotes:
434         cmdline += " --remotes"
435     else:
436         cmdline += " --branches"
438     for line in read_pipe_lines(cmdline):
439         line = line.strip()
441         ## only import to p4/
442         if not line.startswith('p4/') or line == "p4/HEAD":
443             continue
444         branch = line
446         # strip off p4
447         branch = re.sub ("^p4/", "", line)
449         branches[branch] = parseRevision(line)
450     return branches
452 def findUpstreamBranchPoint(head = "HEAD"):
453     branches = p4BranchesInGit()
454     # map from depot-path to branch name
455     branchByDepotPath = {}
456     for branch in branches.keys():
457         tip = branches[branch]
458         log = extractLogMessageFromGitCommit(tip)
459         settings = extractSettingsGitLog(log)
460         if settings.has_key("depot-paths"):
461             paths = ",".join(settings["depot-paths"])
462             branchByDepotPath[paths] = "remotes/p4/" + branch
464     settings = None
465     parent = 0
466     while parent < 65535:
467         commit = head + "~%s" % parent
468         log = extractLogMessageFromGitCommit(commit)
469         settings = extractSettingsGitLog(log)
470         if settings.has_key("depot-paths"):
471             paths = ",".join(settings["depot-paths"])
472             if branchByDepotPath.has_key(paths):
473                 return [branchByDepotPath[paths], settings]
475         parent = parent + 1
477     return ["", settings]
479 def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True):
480     if not silent:
481         print ("Creating/updating branch(es) in %s based on origin branch(es)"
482                % localRefPrefix)
484     originPrefix = "origin/p4/"
486     for line in read_pipe_lines("git rev-parse --symbolic --remotes"):
487         line = line.strip()
488         if (not line.startswith(originPrefix)) or line.endswith("HEAD"):
489             continue
491         headName = line[len(originPrefix):]
492         remoteHead = localRefPrefix + headName
493         originHead = line
495         original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead))
496         if (not original.has_key('depot-paths')
497             or not original.has_key('change')):
498             continue
500         update = False
501         if not gitBranchExists(remoteHead):
502             if verbose:
503                 print "creating %s" % remoteHead
504             update = True
505         else:
506             settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead))
507             if settings.has_key('change') > 0:
508                 if settings['depot-paths'] == original['depot-paths']:
509                     originP4Change = int(original['change'])
510                     p4Change = int(settings['change'])
511                     if originP4Change > p4Change:
512                         print ("%s (%s) is newer than %s (%s). "
513                                "Updating p4 branch from origin."
514                                % (originHead, originP4Change,
515                                   remoteHead, p4Change))
516                         update = True
517                 else:
518                     print ("Ignoring: %s was imported from %s while "
519                            "%s was imported from %s"
520                            % (originHead, ','.join(original['depot-paths']),
521                               remoteHead, ','.join(settings['depot-paths'])))
523         if update:
524             system("git update-ref %s %s" % (remoteHead, originHead))
526 def originP4BranchesExist():
527         return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master")
529 def p4ChangesForPaths(depotPaths, changeRange):
530     assert depotPaths
531     cmd = ['changes']
532     for p in depotPaths:
533         cmd += ["%s...%s" % (p, changeRange)]
534     output = p4_read_pipe_lines(cmd)
536     changes = {}
537     for line in output:
538         changeNum = int(line.split(" ")[1])
539         changes[changeNum] = True
541     changelist = changes.keys()
542     changelist.sort()
543     return changelist
545 def p4PathStartsWith(path, prefix):
546     # This method tries to remedy a potential mixed-case issue:
547     #
548     # If UserA adds  //depot/DirA/file1
549     # and UserB adds //depot/dira/file2
550     #
551     # we may or may not have a problem. If you have core.ignorecase=true,
552     # we treat DirA and dira as the same directory
553     ignorecase = gitConfig("core.ignorecase", "--bool") == "true"
554     if ignorecase:
555         return path.lower().startswith(prefix.lower())
556     return path.startswith(prefix)
558 class Command:
559     def __init__(self):
560         self.usage = "usage: %prog [options]"
561         self.needsGit = True
563 class P4UserMap:
564     def __init__(self):
565         self.userMapFromPerforceServer = False
567     def getUserCacheFilename(self):
568         home = os.environ.get("HOME", os.environ.get("USERPROFILE"))
569         return home + "/.gitp4-usercache.txt"
571     def getUserMapFromPerforceServer(self):
572         if self.userMapFromPerforceServer:
573             return
574         self.users = {}
575         self.emails = {}
577         for output in p4CmdList("users"):
578             if not output.has_key("User"):
579                 continue
580             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
581             self.emails[output["Email"]] = output["User"]
584         s = ''
585         for (key, val) in self.users.items():
586             s += "%s\t%s\n" % (key.expandtabs(1), val.expandtabs(1))
588         open(self.getUserCacheFilename(), "wb").write(s)
589         self.userMapFromPerforceServer = True
591     def loadUserMapFromCache(self):
592         self.users = {}
593         self.userMapFromPerforceServer = False
594         try:
595             cache = open(self.getUserCacheFilename(), "rb")
596             lines = cache.readlines()
597             cache.close()
598             for line in lines:
599                 entry = line.strip().split("\t")
600                 self.users[entry[0]] = entry[1]
601         except IOError:
602             self.getUserMapFromPerforceServer()
604 class P4Debug(Command):
605     def __init__(self):
606         Command.__init__(self)
607         self.options = [
608             optparse.make_option("--verbose", dest="verbose", action="store_true",
609                                  default=False),
610             ]
611         self.description = "A tool to debug the output of p4 -G."
612         self.needsGit = False
613         self.verbose = False
615     def run(self, args):
616         j = 0
617         for output in p4CmdList(args):
618             print 'Element: %d' % j
619             j += 1
620             print output
621         return True
623 class P4RollBack(Command):
624     def __init__(self):
625         Command.__init__(self)
626         self.options = [
627             optparse.make_option("--verbose", dest="verbose", action="store_true"),
628             optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true")
629         ]
630         self.description = "A tool to debug the multi-branch import. Don't use :)"
631         self.verbose = False
632         self.rollbackLocalBranches = False
634     def run(self, args):
635         if len(args) != 1:
636             return False
637         maxChange = int(args[0])
639         if "p4ExitCode" in p4Cmd("changes -m 1"):
640             die("Problems executing p4");
642         if self.rollbackLocalBranches:
643             refPrefix = "refs/heads/"
644             lines = read_pipe_lines("git rev-parse --symbolic --branches")
645         else:
646             refPrefix = "refs/remotes/"
647             lines = read_pipe_lines("git rev-parse --symbolic --remotes")
649         for line in lines:
650             if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"):
651                 line = line.strip()
652                 ref = refPrefix + line
653                 log = extractLogMessageFromGitCommit(ref)
654                 settings = extractSettingsGitLog(log)
656                 depotPaths = settings['depot-paths']
657                 change = settings['change']
659                 changed = False
661                 if len(p4Cmd("changes -m 1 "  + ' '.join (['%s...@%s' % (p, maxChange)
662                                                            for p in depotPaths]))) == 0:
663                     print "Branch %s did not exist at change %s, deleting." % (ref, maxChange)
664                     system("git update-ref -d %s `git rev-parse %s`" % (ref, ref))
665                     continue
667                 while change and int(change) > maxChange:
668                     changed = True
669                     if self.verbose:
670                         print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange)
671                     system("git update-ref %s \"%s^\"" % (ref, ref))
672                     log = extractLogMessageFromGitCommit(ref)
673                     settings =  extractSettingsGitLog(log)
676                     depotPaths = settings['depot-paths']
677                     change = settings['change']
679                 if changed:
680                     print "%s rewound to %s" % (ref, change)
682         return True
684 class P4Submit(Command, P4UserMap):
685     def __init__(self):
686         Command.__init__(self)
687         P4UserMap.__init__(self)
688         self.options = [
689                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
690                 optparse.make_option("--origin", dest="origin"),
691                 optparse.make_option("-M", dest="detectRenames", action="store_true"),
692                 # preserve the user, requires relevant p4 permissions
693                 optparse.make_option("--preserve-user", dest="preserveUser", action="store_true"),
694         ]
695         self.description = "Submit changes from git to the perforce depot."
696         self.usage += " [name of git branch to submit into perforce depot]"
697         self.interactive = True
698         self.origin = ""
699         self.detectRenames = False
700         self.verbose = False
701         self.preserveUser = gitConfig("git-p4.preserveUser").lower() == "true"
702         self.isWindows = (platform.system() == "Windows")
703         self.myP4UserId = None
705     def check(self):
706         if len(p4CmdList("opened ...")) > 0:
707             die("You have files opened with perforce! Close them before starting the sync.")
709     # replaces everything between 'Description:' and the next P4 submit template field with the
710     # commit message
711     def prepareLogMessage(self, template, message):
712         result = ""
714         inDescriptionSection = False
716         for line in template.split("\n"):
717             if line.startswith("#"):
718                 result += line + "\n"
719                 continue
721             if inDescriptionSection:
722                 if line.startswith("Files:") or line.startswith("Jobs:"):
723                     inDescriptionSection = False
724                 else:
725                     continue
726             else:
727                 if line.startswith("Description:"):
728                     inDescriptionSection = True
729                     line += "\n"
730                     for messageLine in message.split("\n"):
731                         line += "\t" + messageLine + "\n"
733             result += line + "\n"
735         return result
737     def p4UserForCommit(self,id):
738         # Return the tuple (perforce user,git email) for a given git commit id
739         self.getUserMapFromPerforceServer()
740         gitEmail = read_pipe("git log --max-count=1 --format='%%ae' %s" % id)
741         gitEmail = gitEmail.strip()
742         if not self.emails.has_key(gitEmail):
743             return (None,gitEmail)
744         else:
745             return (self.emails[gitEmail],gitEmail)
747     def checkValidP4Users(self,commits):
748         # check if any git authors cannot be mapped to p4 users
749         for id in commits:
750             (user,email) = self.p4UserForCommit(id)
751             if not user:
752                 msg = "Cannot find p4 user for email %s in commit %s." % (email, id)
753                 if gitConfig('git-p4.allowMissingP4Users').lower() == "true":
754                     print "%s" % msg
755                 else:
756                     die("Error: %s\nSet git-p4.allowMissingP4Users to true to allow this." % msg)
758     def lastP4Changelist(self):
759         # Get back the last changelist number submitted in this client spec. This
760         # then gets used to patch up the username in the change. If the same
761         # client spec is being used by multiple processes then this might go
762         # wrong.
763         results = p4CmdList("client -o")        # find the current client
764         client = None
765         for r in results:
766             if r.has_key('Client'):
767                 client = r['Client']
768                 break
769         if not client:
770             die("could not get client spec")
771         results = p4CmdList(["changes", "-c", client, "-m", "1"])
772         for r in results:
773             if r.has_key('change'):
774                 return r['change']
775         die("Could not get changelist number for last submit - cannot patch up user details")
777     def modifyChangelistUser(self, changelist, newUser):
778         # fixup the user field of a changelist after it has been submitted.
779         changes = p4CmdList("change -o %s" % changelist)
780         if len(changes) != 1:
781             die("Bad output from p4 change modifying %s to user %s" %
782                 (changelist, newUser))
784         c = changes[0]
785         if c['User'] == newUser: return   # nothing to do
786         c['User'] = newUser
787         input = marshal.dumps(c)
789         result = p4CmdList("change -f -i", stdin=input)
790         for r in result:
791             if r.has_key('code'):
792                 if r['code'] == 'error':
793                     die("Could not modify user field of changelist %s to %s:%s" % (changelist, newUser, r['data']))
794             if r.has_key('data'):
795                 print("Updated user field for changelist %s to %s" % (changelist, newUser))
796                 return
797         die("Could not modify user field of changelist %s to %s" % (changelist, newUser))
799     def canChangeChangelists(self):
800         # check to see if we have p4 admin or super-user permissions, either of
801         # which are required to modify changelists.
802         results = p4CmdList("protects %s" % self.depotPath)
803         for r in results:
804             if r.has_key('perm'):
805                 if r['perm'] == 'admin':
806                     return 1
807                 if r['perm'] == 'super':
808                     return 1
809         return 0
811     def p4UserId(self):
812         if self.myP4UserId:
813             return self.myP4UserId
815         results = p4CmdList("user -o")
816         for r in results:
817             if r.has_key('User'):
818                 self.myP4UserId = r['User']
819                 return r['User']
820         die("Could not find your p4 user id")
822     def p4UserIsMe(self, p4User):
823         # return True if the given p4 user is actually me
824         me = self.p4UserId()
825         if not p4User or p4User != me:
826             return False
827         else:
828             return True
830     def prepareSubmitTemplate(self):
831         # remove lines in the Files section that show changes to files outside the depot path we're committing into
832         template = ""
833         inFilesSection = False
834         for line in p4_read_pipe_lines(['change', '-o']):
835             if line.endswith("\r\n"):
836                 line = line[:-2] + "\n"
837             if inFilesSection:
838                 if line.startswith("\t"):
839                     # path starts and ends with a tab
840                     path = line[1:]
841                     lastTab = path.rfind("\t")
842                     if lastTab != -1:
843                         path = path[:lastTab]
844                         if not p4PathStartsWith(path, self.depotPath):
845                             continue
846                 else:
847                     inFilesSection = False
848             else:
849                 if line.startswith("Files:"):
850                     inFilesSection = True
852             template += line
854         return template
856     def edit_template(self, template_file):
857         """Invoke the editor to let the user change the submission
858            message.  Return true if okay to continue with the submit."""
860         # if configured to skip the editing part, just submit
861         if gitConfig("git-p4.skipSubmitEdit") == "true":
862             return True
864         # look at the modification time, to check later if the user saved
865         # the file
866         mtime = os.stat(template_file).st_mtime
868         # invoke the editor
869         if os.environ.has_key("P4EDITOR"):
870             editor = os.environ.get("P4EDITOR")
871         else:
872             editor = read_pipe("git var GIT_EDITOR").strip()
873         system(editor + " " + template_file)
875         # If the file was not saved, prompt to see if this patch should
876         # be skipped.  But skip this verification step if configured so.
877         if gitConfig("git-p4.skipSubmitEditCheck") == "true":
878             return True
880         # modification time updated means user saved the file
881         if os.stat(template_file).st_mtime > mtime:
882             return True
884         while True:
885             response = raw_input("Submit template unchanged. Submit anyway? [y]es, [n]o (skip this patch) ")
886             if response == 'y':
887                 return True
888             if response == 'n':
889                 return False
891     def applyCommit(self, id):
892         print "Applying %s" % (read_pipe("git log --max-count=1 --pretty=oneline %s" % id))
894         (p4User, gitEmail) = self.p4UserForCommit(id)
896         if not self.detectRenames:
897             # If not explicitly set check the config variable
898             self.detectRenames = gitConfig("git-p4.detectRenames")
900         if self.detectRenames.lower() == "false" or self.detectRenames == "":
901             diffOpts = ""
902         elif self.detectRenames.lower() == "true":
903             diffOpts = "-M"
904         else:
905             diffOpts = "-M%s" % self.detectRenames
907         detectCopies = gitConfig("git-p4.detectCopies")
908         if detectCopies.lower() == "true":
909             diffOpts += " -C"
910         elif detectCopies != "" and detectCopies.lower() != "false":
911             diffOpts += " -C%s" % detectCopies
913         if gitConfig("git-p4.detectCopiesHarder", "--bool") == "true":
914             diffOpts += " --find-copies-harder"
916         diff = read_pipe_lines("git diff-tree -r %s \"%s^\" \"%s\"" % (diffOpts, id, id))
917         filesToAdd = set()
918         filesToDelete = set()
919         editedFiles = set()
920         filesToChangeExecBit = {}
921         for line in diff:
922             diff = parseDiffTreeEntry(line)
923             modifier = diff['status']
924             path = diff['src']
925             if modifier == "M":
926                 p4_edit(path)
927                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
928                     filesToChangeExecBit[path] = diff['dst_mode']
929                 editedFiles.add(path)
930             elif modifier == "A":
931                 filesToAdd.add(path)
932                 filesToChangeExecBit[path] = diff['dst_mode']
933                 if path in filesToDelete:
934                     filesToDelete.remove(path)
935             elif modifier == "D":
936                 filesToDelete.add(path)
937                 if path in filesToAdd:
938                     filesToAdd.remove(path)
939             elif modifier == "C":
940                 src, dest = diff['src'], diff['dst']
941                 p4_integrate(src, dest)
942                 if diff['src_sha1'] != diff['dst_sha1']:
943                     p4_edit(dest)
944                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
945                     p4_edit(dest)
946                     filesToChangeExecBit[dest] = diff['dst_mode']
947                 os.unlink(dest)
948                 editedFiles.add(dest)
949             elif modifier == "R":
950                 src, dest = diff['src'], diff['dst']
951                 p4_integrate(src, dest)
952                 if diff['src_sha1'] != diff['dst_sha1']:
953                     p4_edit(dest)
954                 if isModeExecChanged(diff['src_mode'], diff['dst_mode']):
955                     p4_edit(dest)
956                     filesToChangeExecBit[dest] = diff['dst_mode']
957                 os.unlink(dest)
958                 editedFiles.add(dest)
959                 filesToDelete.add(src)
960             else:
961                 die("unknown modifier %s for %s" % (modifier, path))
963         diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id)
964         patchcmd = diffcmd + " | git apply "
965         tryPatchCmd = patchcmd + "--check -"
966         applyPatchCmd = patchcmd + "--check --apply -"
968         if os.system(tryPatchCmd) != 0:
969             print "Unfortunately applying the change failed!"
970             print "What do you want to do?"
971             response = "x"
972             while response != "s" and response != "a" and response != "w":
973                 response = raw_input("[s]kip this patch / [a]pply the patch forcibly "
974                                      "and with .rej files / [w]rite the patch to a file (patch.txt) ")
975             if response == "s":
976                 print "Skipping! Good luck with the next patches..."
977                 for f in editedFiles:
978                     p4_revert(f)
979                 for f in filesToAdd:
980                     os.remove(f)
981                 return
982             elif response == "a":
983                 os.system(applyPatchCmd)
984                 if len(filesToAdd) > 0:
985                     print "You may also want to call p4 add on the following files:"
986                     print " ".join(filesToAdd)
987                 if len(filesToDelete):
988                     print "The following files should be scheduled for deletion with p4 delete:"
989                     print " ".join(filesToDelete)
990                 die("Please resolve and submit the conflict manually and "
991                     + "continue afterwards with git-p4 submit --continue")
992             elif response == "w":
993                 system(diffcmd + " > patch.txt")
994                 print "Patch saved to patch.txt in %s !" % self.clientPath
995                 die("Please resolve and submit the conflict manually and "
996                     "continue afterwards with git-p4 submit --continue")
998         system(applyPatchCmd)
1000         for f in filesToAdd:
1001             p4_add(f)
1002         for f in filesToDelete:
1003             p4_revert(f)
1004             p4_delete(f)
1006         # Set/clear executable bits
1007         for f in filesToChangeExecBit.keys():
1008             mode = filesToChangeExecBit[f]
1009             setP4ExecBit(f, mode)
1011         logMessage = extractLogMessageFromGitCommit(id)
1012         logMessage = logMessage.strip()
1014         template = self.prepareSubmitTemplate()
1016         if self.interactive:
1017             submitTemplate = self.prepareLogMessage(template, logMessage)
1019             if self.preserveUser:
1020                submitTemplate = submitTemplate + ("\n######## Actual user %s, modified after commit\n" % p4User)
1022             if os.environ.has_key("P4DIFF"):
1023                 del(os.environ["P4DIFF"])
1024             diff = ""
1025             for editedFile in editedFiles:
1026                 diff += p4_read_pipe(['diff', '-du', editedFile])
1028             newdiff = ""
1029             for newFile in filesToAdd:
1030                 newdiff += "==== new file ====\n"
1031                 newdiff += "--- /dev/null\n"
1032                 newdiff += "+++ %s\n" % newFile
1033                 f = open(newFile, "r")
1034                 for line in f.readlines():
1035                     newdiff += "+" + line
1036                 f.close()
1038             if self.checkAuthorship and not self.p4UserIsMe(p4User):
1039                 submitTemplate += "######## git author %s does not match your p4 account.\n" % gitEmail
1040                 submitTemplate += "######## Use git-p4 option --preserve-user to modify authorship\n"
1041                 submitTemplate += "######## Use git-p4 config git-p4.skipUserNameCheck hides this message.\n"
1043             separatorLine = "######## everything below this line is just the diff #######\n"
1045             (handle, fileName) = tempfile.mkstemp()
1046             tmpFile = os.fdopen(handle, "w+")
1047             if self.isWindows:
1048                 submitTemplate = submitTemplate.replace("\n", "\r\n")
1049                 separatorLine = separatorLine.replace("\n", "\r\n")
1050                 newdiff = newdiff.replace("\n", "\r\n")
1051             tmpFile.write(submitTemplate + separatorLine + diff + newdiff)
1052             tmpFile.close()
1054             if self.edit_template(fileName):
1055                 # read the edited message and submit
1056                 tmpFile = open(fileName, "rb")
1057                 message = tmpFile.read()
1058                 tmpFile.close()
1059                 submitTemplate = message[:message.index(separatorLine)]
1060                 if self.isWindows:
1061                     submitTemplate = submitTemplate.replace("\r\n", "\n")
1062                 p4_write_pipe(['submit', '-i'], submitTemplate)
1064                 if self.preserveUser:
1065                     if p4User:
1066                         # Get last changelist number. Cannot easily get it from
1067                         # the submit command output as the output is
1068                         # unmarshalled.
1069                         changelist = self.lastP4Changelist()
1070                         self.modifyChangelistUser(changelist, p4User)
1071             else:
1072                 # skip this patch
1073                 print "Submission cancelled, undoing p4 changes."
1074                 for f in editedFiles:
1075                     p4_revert(f)
1076                 for f in filesToAdd:
1077                     p4_revert(f)
1078                     os.remove(f)
1080             os.remove(fileName)
1081         else:
1082             fileName = "submit.txt"
1083             file = open(fileName, "w+")
1084             file.write(self.prepareLogMessage(template, logMessage))
1085             file.close()
1086             print ("Perforce submit template written as %s. "
1087                    + "Please review/edit and then use p4 submit -i < %s to submit directly!"
1088                    % (fileName, fileName))
1090     def run(self, args):
1091         if len(args) == 0:
1092             self.master = currentGitBranch()
1093             if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master):
1094                 die("Detecting current git branch failed!")
1095         elif len(args) == 1:
1096             self.master = args[0]
1097             if not branchExists(self.master):
1098                 die("Branch %s does not exist" % self.master)
1099         else:
1100             return False
1102         allowSubmit = gitConfig("git-p4.allowSubmit")
1103         if len(allowSubmit) > 0 and not self.master in allowSubmit.split(","):
1104             die("%s is not in git-p4.allowSubmit" % self.master)
1106         [upstream, settings] = findUpstreamBranchPoint()
1107         self.depotPath = settings['depot-paths'][0]
1108         if len(self.origin) == 0:
1109             self.origin = upstream
1111         if self.preserveUser:
1112             if not self.canChangeChangelists():
1113                 die("Cannot preserve user names without p4 super-user or admin permissions")
1115         if self.verbose:
1116             print "Origin branch is " + self.origin
1118         if len(self.depotPath) == 0:
1119             print "Internal error: cannot locate perforce depot path from existing branches"
1120             sys.exit(128)
1122         self.clientPath = p4Where(self.depotPath)
1124         if len(self.clientPath) == 0:
1125             print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath
1126             sys.exit(128)
1128         print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath)
1129         self.oldWorkingDirectory = os.getcwd()
1131         # ensure the clientPath exists
1132         if not os.path.exists(self.clientPath):
1133             os.makedirs(self.clientPath)
1135         chdir(self.clientPath)
1136         print "Synchronizing p4 checkout..."
1137         p4_sync("...")
1138         self.check()
1140         commits = []
1141         for line in read_pipe_lines("git rev-list --no-merges %s..%s" % (self.origin, self.master)):
1142             commits.append(line.strip())
1143         commits.reverse()
1145         if self.preserveUser or (gitConfig("git-p4.skipUserNameCheck") == "true"):
1146             self.checkAuthorship = False
1147         else:
1148             self.checkAuthorship = True
1150         if self.preserveUser:
1151             self.checkValidP4Users(commits)
1153         while len(commits) > 0:
1154             commit = commits[0]
1155             commits = commits[1:]
1156             self.applyCommit(commit)
1157             if not self.interactive:
1158                 break
1160         if len(commits) == 0:
1161             print "All changes applied!"
1162             chdir(self.oldWorkingDirectory)
1164             sync = P4Sync()
1165             sync.run([])
1167             rebase = P4Rebase()
1168             rebase.rebase()
1170         return True
1172 class P4Sync(Command, P4UserMap):
1173     delete_actions = ( "delete", "move/delete", "purge" )
1175     def __init__(self):
1176         Command.__init__(self)
1177         P4UserMap.__init__(self)
1178         self.options = [
1179                 optparse.make_option("--branch", dest="branch"),
1180                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
1181                 optparse.make_option("--changesfile", dest="changesFile"),
1182                 optparse.make_option("--silent", dest="silent", action="store_true"),
1183                 optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"),
1184                 optparse.make_option("--verbose", dest="verbose", action="store_true"),
1185                 optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false",
1186                                      help="Import into refs/heads/ , not refs/remotes"),
1187                 optparse.make_option("--max-changes", dest="maxChanges"),
1188                 optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true',
1189                                      help="Keep entire BRANCH/DIR/SUBDIR prefix during import"),
1190                 optparse.make_option("--use-client-spec", dest="useClientSpec", action='store_true',
1191                                      help="Only sync files that are included in the Perforce Client Spec")
1192         ]
1193         self.description = """Imports from Perforce into a git repository.\n
1194     example:
1195     //depot/my/project/ -- to import the current head
1196     //depot/my/project/@all -- to import everything
1197     //depot/my/project/@1,6 -- to import only from revision 1 to 6
1199     (a ... is not needed in the path p4 specification, it's added implicitly)"""
1201         self.usage += " //depot/path[@revRange]"
1202         self.silent = False
1203         self.createdBranches = set()
1204         self.committedChanges = set()
1205         self.branch = ""
1206         self.detectBranches = False
1207         self.detectLabels = False
1208         self.changesFile = ""
1209         self.syncWithOrigin = True
1210         self.verbose = False
1211         self.importIntoRemotes = True
1212         self.maxChanges = ""
1213         self.isWindows = (platform.system() == "Windows")
1214         self.keepRepoPath = False
1215         self.depotPaths = None
1216         self.p4BranchesInGit = []
1217         self.cloneExclude = []
1218         self.useClientSpec = False
1219         self.clientSpecDirs = []
1220         self.haveSingleFileClientViews = False
1222         if gitConfig("git-p4.syncFromOrigin") == "false":
1223             self.syncWithOrigin = False
1225     #
1226     # P4 wildcards are not allowed in filenames.  P4 complains
1227     # if you simply add them, but you can force it with "-f", in
1228     # which case it translates them into %xx encoding internally.
1229     # Search for and fix just these four characters.  Do % last so
1230     # that fixing it does not inadvertently create new %-escapes.
1231     #
1232     def wildcard_decode(self, path):
1233         # Cannot have * in a filename in windows; untested as to
1234         # what p4 would do in such a case.
1235         if not self.isWindows:
1236             path = path.replace("%2A", "*")
1237         path = path.replace("%23", "#") \
1238                    .replace("%40", "@") \
1239                    .replace("%25", "%")
1240         return path
1242     def extractFilesFromCommit(self, commit):
1243         self.cloneExclude = [re.sub(r"\.\.\.$", "", path)
1244                              for path in self.cloneExclude]
1245         files = []
1246         fnum = 0
1247         while commit.has_key("depotFile%s" % fnum):
1248             path =  commit["depotFile%s" % fnum]
1250             if [p for p in self.cloneExclude
1251                 if p4PathStartsWith(path, p)]:
1252                 found = False
1253             else:
1254                 found = [p for p in self.depotPaths
1255                          if p4PathStartsWith(path, p)]
1256             if not found:
1257                 fnum = fnum + 1
1258                 continue
1260             file = {}
1261             file["path"] = path
1262             file["rev"] = commit["rev%s" % fnum]
1263             file["action"] = commit["action%s" % fnum]
1264             file["type"] = commit["type%s" % fnum]
1265             files.append(file)
1266             fnum = fnum + 1
1267         return files
1269     def stripRepoPath(self, path, prefixes):
1270         if self.useClientSpec:
1272             # if using the client spec, we use the output directory
1273             # specified in the client.  For example, a view
1274             #   //depot/foo/branch/... //client/branch/foo/...
1275             # will end up putting all foo/branch files into
1276             #  branch/foo/
1277             for val in self.clientSpecDirs:
1278                 if self.haveSingleFileClientViews and len(path) == abs(val[1][0]) and path == val[0]:
1279                     # since 'path' is a depot file path, if it matches the LeftMap,
1280                     # then the View is a single file mapping, so use the entire rightMap
1281                     # first two tests above avoid the last == test for common cases
1282                     path =  val[1][1]
1283                     # now strip out the client (//client/...)
1284                     path = re.sub("^(//[^/]+/)", '', path)
1285                     # the rest is local client path
1286                     return path
1288                 if path.startswith(val[0]):
1289                     # replace the depot path with the client path
1290                     path = path.replace(val[0], val[1][1])
1291                     # now strip out the client (//client/...)
1292                     path = re.sub("^(//[^/]+/)", '', path)
1293                     # the rest is all path
1294                     return path
1296         if self.keepRepoPath:
1297             prefixes = [re.sub("^(//[^/]+/).*", r'\1', prefixes[0])]
1299         for p in prefixes:
1300             if p4PathStartsWith(path, p):
1301                 path = path[len(p):]
1303         return path
1305     def splitFilesIntoBranches(self, commit):
1306         branches = {}
1307         fnum = 0
1308         while commit.has_key("depotFile%s" % fnum):
1309             path =  commit["depotFile%s" % fnum]
1310             found = [p for p in self.depotPaths
1311                      if p4PathStartsWith(path, p)]
1312             if not found:
1313                 fnum = fnum + 1
1314                 continue
1316             file = {}
1317             file["path"] = path
1318             file["rev"] = commit["rev%s" % fnum]
1319             file["action"] = commit["action%s" % fnum]
1320             file["type"] = commit["type%s" % fnum]
1321             fnum = fnum + 1
1323             relPath = self.stripRepoPath(path, self.depotPaths)
1325             for branch in self.knownBranches.keys():
1327                 # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2
1328                 if relPath.startswith(branch + "/"):
1329                     if branch not in branches:
1330                         branches[branch] = []
1331                     branches[branch].append(file)
1332                     break
1334         return branches
1336     # output one file from the P4 stream
1337     # - helper for streamP4Files
1339     def streamOneP4File(self, file, contents):
1340         relPath = self.stripRepoPath(file['depotFile'], self.branchPrefixes)
1341         relPath = self.wildcard_decode(relPath)
1342         if verbose:
1343             sys.stderr.write("%s\n" % relPath)
1345         (type_base, type_mods) = split_p4_type(file["type"])
1347         git_mode = "100644"
1348         if "x" in type_mods:
1349             git_mode = "100755"
1350         if type_base == "symlink":
1351             git_mode = "120000"
1352             # p4 print on a symlink contains "target\n"; remove the newline
1353             data = ''.join(contents)
1354             contents = [data[:-1]]
1356         if type_base == "utf16":
1357             # p4 delivers different text in the python output to -G
1358             # than it does when using "print -o", or normal p4 client
1359             # operations.  utf16 is converted to ascii or utf8, perhaps.
1360             # But ascii text saved as -t utf16 is completely mangled.
1361             # Invoke print -o to get the real contents.
1362             text = p4_read_pipe(['print', '-q', '-o', '-', file['depotFile']])
1363             contents = [ text ]
1365         if type_base == "apple":
1366             # Apple filetype files will be streamed as a concatenation of
1367             # its appledouble header and the contents.  This is useless
1368             # on both macs and non-macs.  If using "print -q -o xx", it
1369             # will create "xx" with the data, and "%xx" with the header.
1370             # This is also not very useful.
1371             #
1372             # Ideally, someday, this script can learn how to generate
1373             # appledouble files directly and import those to git, but
1374             # non-mac machines can never find a use for apple filetype.
1375             print "\nIgnoring apple filetype file %s" % file['depotFile']
1376             return
1378         # Perhaps windows wants unicode, utf16 newlines translated too;
1379         # but this is not doing it.
1380         if self.isWindows and type_base == "text":
1381             mangled = []
1382             for data in contents:
1383                 data = data.replace("\r\n", "\n")
1384                 mangled.append(data)
1385             contents = mangled
1387         # Note that we do not try to de-mangle keywords on utf16 files,
1388         # even though in theory somebody may want that.
1389         if type_base in ("text", "unicode", "binary"):
1390             if "ko" in type_mods:
1391                 text = ''.join(contents)
1392                 text = re.sub(r'\$(Id|Header):[^$]*\$', r'$\1$', text)
1393                 contents = [ text ]
1394             elif "k" in type_mods:
1395                 text = ''.join(contents)
1396                 text = re.sub(r'\$(Id|Header|Author|Date|DateTime|Change|File|Revision):[^$]*\$', r'$\1$', text)
1397                 contents = [ text ]
1399         self.gitStream.write("M %s inline %s\n" % (git_mode, relPath))
1401         # total length...
1402         length = 0
1403         for d in contents:
1404             length = length + len(d)
1406         self.gitStream.write("data %d\n" % length)
1407         for d in contents:
1408             self.gitStream.write(d)
1409         self.gitStream.write("\n")
1411     def streamOneP4Deletion(self, file):
1412         relPath = self.stripRepoPath(file['path'], self.branchPrefixes)
1413         if verbose:
1414             sys.stderr.write("delete %s\n" % relPath)
1415         self.gitStream.write("D %s\n" % relPath)
1417     # handle another chunk of streaming data
1418     def streamP4FilesCb(self, marshalled):
1420         if marshalled.has_key('depotFile') and self.stream_have_file_info:
1421             # start of a new file - output the old one first
1422             self.streamOneP4File(self.stream_file, self.stream_contents)
1423             self.stream_file = {}
1424             self.stream_contents = []
1425             self.stream_have_file_info = False
1427         # pick up the new file information... for the
1428         # 'data' field we need to append to our array
1429         for k in marshalled.keys():
1430             if k == 'data':
1431                 self.stream_contents.append(marshalled['data'])
1432             else:
1433                 self.stream_file[k] = marshalled[k]
1435         self.stream_have_file_info = True
1437     # Stream directly from "p4 files" into "git fast-import"
1438     def streamP4Files(self, files):
1439         filesForCommit = []
1440         filesToRead = []
1441         filesToDelete = []
1443         for f in files:
1444             includeFile = True
1445             for val in self.clientSpecDirs:
1446                 if f['path'].startswith(val[0]):
1447                     if val[1][0] <= 0:
1448                         includeFile = False
1449                     break
1451             if includeFile:
1452                 filesForCommit.append(f)
1453                 if f['action'] in self.delete_actions:
1454                     filesToDelete.append(f)
1455                 else:
1456                     filesToRead.append(f)
1458         # deleted files...
1459         for f in filesToDelete:
1460             self.streamOneP4Deletion(f)
1462         if len(filesToRead) > 0:
1463             self.stream_file = {}
1464             self.stream_contents = []
1465             self.stream_have_file_info = False
1467             # curry self argument
1468             def streamP4FilesCbSelf(entry):
1469                 self.streamP4FilesCb(entry)
1471             fileArgs = ['%s#%s' % (f['path'], f['rev']) for f in filesToRead]
1473             p4CmdList(["-x", "-", "print"],
1474                       stdin=fileArgs,
1475                       cb=streamP4FilesCbSelf)
1477             # do the last chunk
1478             if self.stream_file.has_key('depotFile'):
1479                 self.streamOneP4File(self.stream_file, self.stream_contents)
1481     def commit(self, details, files, branch, branchPrefixes, parent = ""):
1482         epoch = details["time"]
1483         author = details["user"]
1484         self.branchPrefixes = branchPrefixes
1486         if self.verbose:
1487             print "commit into %s" % branch
1489         # start with reading files; if that fails, we should not
1490         # create a commit.
1491         new_files = []
1492         for f in files:
1493             if [p for p in branchPrefixes if p4PathStartsWith(f['path'], p)]:
1494                 new_files.append (f)
1495             else:
1496                 sys.stderr.write("Ignoring file outside of prefix: %s\n" % f['path'])
1498         self.gitStream.write("commit %s\n" % branch)
1499 #        gitStream.write("mark :%s\n" % details["change"])
1500         self.committedChanges.add(int(details["change"]))
1501         committer = ""
1502         if author not in self.users:
1503             self.getUserMapFromPerforceServer()
1504         if author in self.users:
1505             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
1506         else:
1507             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
1509         self.gitStream.write("committer %s\n" % committer)
1511         self.gitStream.write("data <<EOT\n")
1512         self.gitStream.write(details["desc"])
1513         self.gitStream.write("\n[git-p4: depot-paths = \"%s\": change = %s"
1514                              % (','.join (branchPrefixes), details["change"]))
1515         if len(details['options']) > 0:
1516             self.gitStream.write(": options = %s" % details['options'])
1517         self.gitStream.write("]\nEOT\n\n")
1519         if len(parent) > 0:
1520             if self.verbose:
1521                 print "parent %s" % parent
1522             self.gitStream.write("from %s\n" % parent)
1524         self.streamP4Files(new_files)
1525         self.gitStream.write("\n")
1527         change = int(details["change"])
1529         if self.labels.has_key(change):
1530             label = self.labels[change]
1531             labelDetails = label[0]
1532             labelRevisions = label[1]
1533             if self.verbose:
1534                 print "Change %s is labelled %s" % (change, labelDetails)
1536             files = p4CmdList(["files"] + ["%s...@%s" % (p, change)
1537                                                     for p in branchPrefixes])
1539             if len(files) == len(labelRevisions):
1541                 cleanedFiles = {}
1542                 for info in files:
1543                     if info["action"] in self.delete_actions:
1544                         continue
1545                     cleanedFiles[info["depotFile"]] = info["rev"]
1547                 if cleanedFiles == labelRevisions:
1548                     self.gitStream.write("tag tag_%s\n" % labelDetails["label"])
1549                     self.gitStream.write("from %s\n" % branch)
1551                     owner = labelDetails["Owner"]
1552                     tagger = ""
1553                     if author in self.users:
1554                         tagger = "%s %s %s" % (self.users[owner], epoch, self.tz)
1555                     else:
1556                         tagger = "%s <a@b> %s %s" % (owner, epoch, self.tz)
1557                     self.gitStream.write("tagger %s\n" % tagger)
1558                     self.gitStream.write("data <<EOT\n")
1559                     self.gitStream.write(labelDetails["Description"])
1560                     self.gitStream.write("EOT\n\n")
1562                 else:
1563                     if not self.silent:
1564                         print ("Tag %s does not match with change %s: files do not match."
1565                                % (labelDetails["label"], change))
1567             else:
1568                 if not self.silent:
1569                     print ("Tag %s does not match with change %s: file count is different."
1570                            % (labelDetails["label"], change))
1572     def getLabels(self):
1573         self.labels = {}
1575         l = p4CmdList("labels %s..." % ' '.join (self.depotPaths))
1576         if len(l) > 0 and not self.silent:
1577             print "Finding files belonging to labels in %s" % `self.depotPaths`
1579         for output in l:
1580             label = output["label"]
1581             revisions = {}
1582             newestChange = 0
1583             if self.verbose:
1584                 print "Querying files for label %s" % label
1585             for file in p4CmdList(["files"] +
1586                                       ["%s...@%s" % (p, label)
1587                                           for p in self.depotPaths]):
1588                 revisions[file["depotFile"]] = file["rev"]
1589                 change = int(file["change"])
1590                 if change > newestChange:
1591                     newestChange = change
1593             self.labels[newestChange] = [output, revisions]
1595         if self.verbose:
1596             print "Label changes: %s" % self.labels.keys()
1598     def guessProjectName(self):
1599         for p in self.depotPaths:
1600             if p.endswith("/"):
1601                 p = p[:-1]
1602             p = p[p.strip().rfind("/") + 1:]
1603             if not p.endswith("/"):
1604                p += "/"
1605             return p
1607     def getBranchMapping(self):
1608         lostAndFoundBranches = set()
1610         user = gitConfig("git-p4.branchUser")
1611         if len(user) > 0:
1612             command = "branches -u %s" % user
1613         else:
1614             command = "branches"
1616         for info in p4CmdList(command):
1617             details = p4Cmd("branch -o %s" % info["branch"])
1618             viewIdx = 0
1619             while details.has_key("View%s" % viewIdx):
1620                 paths = details["View%s" % viewIdx].split(" ")
1621                 viewIdx = viewIdx + 1
1622                 # require standard //depot/foo/... //depot/bar/... mapping
1623                 if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."):
1624                     continue
1625                 source = paths[0]
1626                 destination = paths[1]
1627                 ## HACK
1628                 if p4PathStartsWith(source, self.depotPaths[0]) and p4PathStartsWith(destination, self.depotPaths[0]):
1629                     source = source[len(self.depotPaths[0]):-4]
1630                     destination = destination[len(self.depotPaths[0]):-4]
1632                     if destination in self.knownBranches:
1633                         if not self.silent:
1634                             print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination)
1635                             print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination)
1636                         continue
1638                     self.knownBranches[destination] = source
1640                     lostAndFoundBranches.discard(destination)
1642                     if source not in self.knownBranches:
1643                         lostAndFoundBranches.add(source)
1645         # Perforce does not strictly require branches to be defined, so we also
1646         # check git config for a branch list.
1647         #
1648         # Example of branch definition in git config file:
1649         # [git-p4]
1650         #   branchList=main:branchA
1651         #   branchList=main:branchB
1652         #   branchList=branchA:branchC
1653         configBranches = gitConfigList("git-p4.branchList")
1654         for branch in configBranches:
1655             if branch:
1656                 (source, destination) = branch.split(":")
1657                 self.knownBranches[destination] = source
1659                 lostAndFoundBranches.discard(destination)
1661                 if source not in self.knownBranches:
1662                     lostAndFoundBranches.add(source)
1665         for branch in lostAndFoundBranches:
1666             self.knownBranches[branch] = branch
1668     def getBranchMappingFromGitBranches(self):
1669         branches = p4BranchesInGit(self.importIntoRemotes)
1670         for branch in branches.keys():
1671             if branch == "master":
1672                 branch = "main"
1673             else:
1674                 branch = branch[len(self.projectName):]
1675             self.knownBranches[branch] = branch
1677     def listExistingP4GitBranches(self):
1678         # branches holds mapping from name to commit
1679         branches = p4BranchesInGit(self.importIntoRemotes)
1680         self.p4BranchesInGit = branches.keys()
1681         for branch in branches.keys():
1682             self.initialParents[self.refPrefix + branch] = branches[branch]
1684     def updateOptionDict(self, d):
1685         option_keys = {}
1686         if self.keepRepoPath:
1687             option_keys['keepRepoPath'] = 1
1689         d["options"] = ' '.join(sorted(option_keys.keys()))
1691     def readOptions(self, d):
1692         self.keepRepoPath = (d.has_key('options')
1693                              and ('keepRepoPath' in d['options']))
1695     def gitRefForBranch(self, branch):
1696         if branch == "main":
1697             return self.refPrefix + "master"
1699         if len(branch) <= 0:
1700             return branch
1702         return self.refPrefix + self.projectName + branch
1704     def gitCommitByP4Change(self, ref, change):
1705         if self.verbose:
1706             print "looking in ref " + ref + " for change %s using bisect..." % change
1708         earliestCommit = ""
1709         latestCommit = parseRevision(ref)
1711         while True:
1712             if self.verbose:
1713                 print "trying: earliest %s latest %s" % (earliestCommit, latestCommit)
1714             next = read_pipe("git rev-list --bisect %s %s" % (latestCommit, earliestCommit)).strip()
1715             if len(next) == 0:
1716                 if self.verbose:
1717                     print "argh"
1718                 return ""
1719             log = extractLogMessageFromGitCommit(next)
1720             settings = extractSettingsGitLog(log)
1721             currentChange = int(settings['change'])
1722             if self.verbose:
1723                 print "current change %s" % currentChange
1725             if currentChange == change:
1726                 if self.verbose:
1727                     print "found %s" % next
1728                 return next
1730             if currentChange < change:
1731                 earliestCommit = "^%s" % next
1732             else:
1733                 latestCommit = "%s" % next
1735         return ""
1737     def importNewBranch(self, branch, maxChange):
1738         # make fast-import flush all changes to disk and update the refs using the checkpoint
1739         # command so that we can try to find the branch parent in the git history
1740         self.gitStream.write("checkpoint\n\n");
1741         self.gitStream.flush();
1742         branchPrefix = self.depotPaths[0] + branch + "/"
1743         range = "@1,%s" % maxChange
1744         #print "prefix" + branchPrefix
1745         changes = p4ChangesForPaths([branchPrefix], range)
1746         if len(changes) <= 0:
1747             return False
1748         firstChange = changes[0]
1749         #print "first change in branch: %s" % firstChange
1750         sourceBranch = self.knownBranches[branch]
1751         sourceDepotPath = self.depotPaths[0] + sourceBranch
1752         sourceRef = self.gitRefForBranch(sourceBranch)
1753         #print "source " + sourceBranch
1755         branchParentChange = int(p4Cmd("changes -m 1 %s...@1,%s" % (sourceDepotPath, firstChange))["change"])
1756         #print "branch parent: %s" % branchParentChange
1757         gitParent = self.gitCommitByP4Change(sourceRef, branchParentChange)
1758         if len(gitParent) > 0:
1759             self.initialParents[self.gitRefForBranch(branch)] = gitParent
1760             #print "parent git commit: %s" % gitParent
1762         self.importChanges(changes)
1763         return True
1765     def importChanges(self, changes):
1766         cnt = 1
1767         for change in changes:
1768             description = p4Cmd("describe %s" % change)
1769             self.updateOptionDict(description)
1771             if not self.silent:
1772                 sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
1773                 sys.stdout.flush()
1774             cnt = cnt + 1
1776             try:
1777                 if self.detectBranches:
1778                     branches = self.splitFilesIntoBranches(description)
1779                     for branch in branches.keys():
1780                         ## HACK  --hwn
1781                         branchPrefix = self.depotPaths[0] + branch + "/"
1783                         parent = ""
1785                         filesForCommit = branches[branch]
1787                         if self.verbose:
1788                             print "branch is %s" % branch
1790                         self.updatedBranches.add(branch)
1792                         if branch not in self.createdBranches:
1793                             self.createdBranches.add(branch)
1794                             parent = self.knownBranches[branch]
1795                             if parent == branch:
1796                                 parent = ""
1797                             else:
1798                                 fullBranch = self.projectName + branch
1799                                 if fullBranch not in self.p4BranchesInGit:
1800                                     if not self.silent:
1801                                         print("\n    Importing new branch %s" % fullBranch);
1802                                     if self.importNewBranch(branch, change - 1):
1803                                         parent = ""
1804                                         self.p4BranchesInGit.append(fullBranch)
1805                                     if not self.silent:
1806                                         print("\n    Resuming with change %s" % change);
1808                                 if self.verbose:
1809                                     print "parent determined through known branches: %s" % parent
1811                         branch = self.gitRefForBranch(branch)
1812                         parent = self.gitRefForBranch(parent)
1814                         if self.verbose:
1815                             print "looking for initial parent for %s; current parent is %s" % (branch, parent)
1817                         if len(parent) == 0 and branch in self.initialParents:
1818                             parent = self.initialParents[branch]
1819                             del self.initialParents[branch]
1821                         self.commit(description, filesForCommit, branch, [branchPrefix], parent)
1822                 else:
1823                     files = self.extractFilesFromCommit(description)
1824                     self.commit(description, files, self.branch, self.depotPaths,
1825                                 self.initialParent)
1826                     self.initialParent = ""
1827             except IOError:
1828                 print self.gitError.read()
1829                 sys.exit(1)
1831     def importHeadRevision(self, revision):
1832         print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), revision, self.branch)
1834         details = {}
1835         details["user"] = "git perforce import user"
1836         details["desc"] = ("Initial import of %s from the state at revision %s\n"
1837                            % (' '.join(self.depotPaths), revision))
1838         details["change"] = revision
1839         newestRevision = 0
1841         fileCnt = 0
1842         fileArgs = ["%s...%s" % (p,revision) for p in self.depotPaths]
1844         for info in p4CmdList(["files"] + fileArgs):
1846             if 'code' in info and info['code'] == 'error':
1847                 sys.stderr.write("p4 returned an error: %s\n"
1848                                  % info['data'])
1849                 if info['data'].find("must refer to client") >= 0:
1850                     sys.stderr.write("This particular p4 error is misleading.\n")
1851                     sys.stderr.write("Perhaps the depot path was misspelled.\n");
1852                     sys.stderr.write("Depot path:  %s\n" % " ".join(self.depotPaths))
1853                 sys.exit(1)
1854             if 'p4ExitCode' in info:
1855                 sys.stderr.write("p4 exitcode: %s\n" % info['p4ExitCode'])
1856                 sys.exit(1)
1859             change = int(info["change"])
1860             if change > newestRevision:
1861                 newestRevision = change
1863             if info["action"] in self.delete_actions:
1864                 # don't increase the file cnt, otherwise details["depotFile123"] will have gaps!
1865                 #fileCnt = fileCnt + 1
1866                 continue
1868             for prop in ["depotFile", "rev", "action", "type" ]:
1869                 details["%s%s" % (prop, fileCnt)] = info[prop]
1871             fileCnt = fileCnt + 1
1873         details["change"] = newestRevision
1875         # Use time from top-most change so that all git-p4 clones of
1876         # the same p4 repo have the same commit SHA1s.
1877         res = p4CmdList("describe -s %d" % newestRevision)
1878         newestTime = None
1879         for r in res:
1880             if r.has_key('time'):
1881                 newestTime = int(r['time'])
1882         if newestTime is None:
1883             die("\"describe -s\" on newest change %d did not give a time")
1884         details["time"] = newestTime
1886         self.updateOptionDict(details)
1887         try:
1888             self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPaths)
1889         except IOError:
1890             print "IO error with git fast-import. Is your git version recent enough?"
1891             print self.gitError.read()
1894     def getClientSpec(self):
1895         specList = p4CmdList( "client -o" )
1896         temp = {}
1897         for entry in specList:
1898             for k,v in entry.iteritems():
1899                 if k.startswith("View"):
1901                     # p4 has these %%1 to %%9 arguments in specs to
1902                     # reorder paths; which we can't handle (yet :)
1903                     if re.search('%%\d', v) != None:
1904                         print "Sorry, can't handle %%n arguments in client specs"
1905                         sys.exit(1)
1906                     if re.search('\*', v) != None:
1907                         print "Sorry, can't handle * mappings in client specs"
1908                         sys.exit(1)
1910                     if v.startswith('"'):
1911                         start = 1
1912                     else:
1913                         start = 0
1914                     index = v.find("...")
1916                     # save the "client view"; i.e the RHS of the view
1917                     # line that tells the client where to put the
1918                     # files for this view.
1920                     # check for individual file mappings - those which have no '...'
1921                     if  index < 0 :
1922                         v,cv = v.strip().split()
1923                         v = v[start:]
1924                         self.haveSingleFileClientViews = True
1925                     else:
1926                         cv = v[index+3:].strip() # +3 to remove previous '...'
1927                         cv=cv[:-3]
1928                         v = v[start:index]
1930                     # now save the view; +index means included, -index
1931                     # means it should be filtered out.
1932                     if v.startswith("-"):
1933                         v = v[1:]
1934                         include = -len(v)
1935                     else:
1936                         include = len(v)
1938                     # store the View #number for sorting
1939                     # and the View string itself (this last for documentation)
1940                     temp[v] = (include, cv, int(k[4:]),k)
1942         self.clientSpecDirs = temp.items()
1943         # Perforce ViewNN with higher #numbers override those with lower
1944         # reverse sort on the View #number
1945         self.clientSpecDirs.sort( lambda x, y:  y[1][2] -  x[1][2]  )
1946         if self.verbose:
1947             for val in self.clientSpecDirs:
1948                     print "clientSpecDirs: %s %s" % (val[0],val[1])
1950     def run(self, args):
1951         self.depotPaths = []
1952         self.changeRange = ""
1953         self.initialParent = ""
1954         self.previousDepotPaths = []
1956         # map from branch depot path to parent branch
1957         self.knownBranches = {}
1958         self.initialParents = {}
1959         self.hasOrigin = originP4BranchesExist()
1960         if not self.syncWithOrigin:
1961             self.hasOrigin = False
1963         if self.importIntoRemotes:
1964             self.refPrefix = "refs/remotes/p4/"
1965         else:
1966             self.refPrefix = "refs/heads/p4/"
1968         if self.syncWithOrigin and self.hasOrigin:
1969             if not self.silent:
1970                 print "Syncing with origin first by calling git fetch origin"
1971             system("git fetch origin")
1973         if len(self.branch) == 0:
1974             self.branch = self.refPrefix + "master"
1975             if gitBranchExists("refs/heads/p4") and self.importIntoRemotes:
1976                 system("git update-ref %s refs/heads/p4" % self.branch)
1977                 system("git branch -D p4");
1978             # create it /after/ importing, when master exists
1979             if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch):
1980                 system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch))
1982         if not self.useClientSpec:
1983             if gitConfig("git-p4.useclientspec", "--bool") == "true":
1984                 self.useClientSpec = True
1985         if self.useClientSpec:
1986             self.getClientSpec()
1988         # TODO: should always look at previous commits,
1989         # merge with previous imports, if possible.
1990         if args == []:
1991             if self.hasOrigin:
1992                 createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent)
1993             self.listExistingP4GitBranches()
1995             if len(self.p4BranchesInGit) > 1:
1996                 if not self.silent:
1997                     print "Importing from/into multiple branches"
1998                 self.detectBranches = True
2000             if self.verbose:
2001                 print "branches: %s" % self.p4BranchesInGit
2003             p4Change = 0
2004             for branch in self.p4BranchesInGit:
2005                 logMsg =  extractLogMessageFromGitCommit(self.refPrefix + branch)
2007                 settings = extractSettingsGitLog(logMsg)
2009                 self.readOptions(settings)
2010                 if (settings.has_key('depot-paths')
2011                     and settings.has_key ('change')):
2012                     change = int(settings['change']) + 1
2013                     p4Change = max(p4Change, change)
2015                     depotPaths = sorted(settings['depot-paths'])
2016                     if self.previousDepotPaths == []:
2017                         self.previousDepotPaths = depotPaths
2018                     else:
2019                         paths = []
2020                         for (prev, cur) in zip(self.previousDepotPaths, depotPaths):
2021                             prev_list = prev.split("/")
2022                             cur_list = cur.split("/")
2023                             for i in range(0, min(len(cur_list), len(prev_list))):
2024                                 if cur_list[i] <> prev_list[i]:
2025                                     i = i - 1
2026                                     break
2028                             paths.append ("/".join(cur_list[:i + 1]))
2030                         self.previousDepotPaths = paths
2032             if p4Change > 0:
2033                 self.depotPaths = sorted(self.previousDepotPaths)
2034                 self.changeRange = "@%s,#head" % p4Change
2035                 if not self.detectBranches:
2036                     self.initialParent = parseRevision(self.branch)
2037                 if not self.silent and not self.detectBranches:
2038                     print "Performing incremental import into %s git branch" % self.branch
2040         if not self.branch.startswith("refs/"):
2041             self.branch = "refs/heads/" + self.branch
2043         if len(args) == 0 and self.depotPaths:
2044             if not self.silent:
2045                 print "Depot paths: %s" % ' '.join(self.depotPaths)
2046         else:
2047             if self.depotPaths and self.depotPaths != args:
2048                 print ("previous import used depot path %s and now %s was specified. "
2049                        "This doesn't work!" % (' '.join (self.depotPaths),
2050                                                ' '.join (args)))
2051                 sys.exit(1)
2053             self.depotPaths = sorted(args)
2055         revision = ""
2056         self.users = {}
2058         # Make sure no revision specifiers are used when --changesfile
2059         # is specified.
2060         bad_changesfile = False
2061         if len(self.changesFile) > 0:
2062             for p in self.depotPaths:
2063                 if p.find("@") >= 0 or p.find("#") >= 0:
2064                     bad_changesfile = True
2065                     break
2066         if bad_changesfile:
2067             die("Option --changesfile is incompatible with revision specifiers")
2069         newPaths = []
2070         for p in self.depotPaths:
2071             if p.find("@") != -1:
2072                 atIdx = p.index("@")
2073                 self.changeRange = p[atIdx:]
2074                 if self.changeRange == "@all":
2075                     self.changeRange = ""
2076                 elif ',' not in self.changeRange:
2077                     revision = self.changeRange
2078                     self.changeRange = ""
2079                 p = p[:atIdx]
2080             elif p.find("#") != -1:
2081                 hashIdx = p.index("#")
2082                 revision = p[hashIdx:]
2083                 p = p[:hashIdx]
2084             elif self.previousDepotPaths == []:
2085                 # pay attention to changesfile, if given, else import
2086                 # the entire p4 tree at the head revision
2087                 if len(self.changesFile) == 0:
2088                     revision = "#head"
2090             p = re.sub ("\.\.\.$", "", p)
2091             if not p.endswith("/"):
2092                 p += "/"
2094             newPaths.append(p)
2096         self.depotPaths = newPaths
2099         self.loadUserMapFromCache()
2100         self.labels = {}
2101         if self.detectLabels:
2102             self.getLabels();
2104         if self.detectBranches:
2105             ## FIXME - what's a P4 projectName ?
2106             self.projectName = self.guessProjectName()
2108             if self.hasOrigin:
2109                 self.getBranchMappingFromGitBranches()
2110             else:
2111                 self.getBranchMapping()
2112             if self.verbose:
2113                 print "p4-git branches: %s" % self.p4BranchesInGit
2114                 print "initial parents: %s" % self.initialParents
2115             for b in self.p4BranchesInGit:
2116                 if b != "master":
2118                     ## FIXME
2119                     b = b[len(self.projectName):]
2120                 self.createdBranches.add(b)
2122         self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60))
2124         importProcess = subprocess.Popen(["git", "fast-import"],
2125                                          stdin=subprocess.PIPE, stdout=subprocess.PIPE,
2126                                          stderr=subprocess.PIPE);
2127         self.gitOutput = importProcess.stdout
2128         self.gitStream = importProcess.stdin
2129         self.gitError = importProcess.stderr
2131         if revision:
2132             self.importHeadRevision(revision)
2133         else:
2134             changes = []
2136             if len(self.changesFile) > 0:
2137                 output = open(self.changesFile).readlines()
2138                 changeSet = set()
2139                 for line in output:
2140                     changeSet.add(int(line))
2142                 for change in changeSet:
2143                     changes.append(change)
2145                 changes.sort()
2146             else:
2147                 # catch "git-p4 sync" with no new branches, in a repo that
2148                 # does not have any existing git-p4 branches
2149                 if len(args) == 0 and not self.p4BranchesInGit:
2150                     die("No remote p4 branches.  Perhaps you never did \"git p4 clone\" in here.");
2151                 if self.verbose:
2152                     print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths),
2153                                                               self.changeRange)
2154                 changes = p4ChangesForPaths(self.depotPaths, self.changeRange)
2156                 if len(self.maxChanges) > 0:
2157                     changes = changes[:min(int(self.maxChanges), len(changes))]
2159             if len(changes) == 0:
2160                 if not self.silent:
2161                     print "No changes to import!"
2162                 return True
2164             if not self.silent and not self.detectBranches:
2165                 print "Import destination: %s" % self.branch
2167             self.updatedBranches = set()
2169             self.importChanges(changes)
2171             if not self.silent:
2172                 print ""
2173                 if len(self.updatedBranches) > 0:
2174                     sys.stdout.write("Updated branches: ")
2175                     for b in self.updatedBranches:
2176                         sys.stdout.write("%s " % b)
2177                     sys.stdout.write("\n")
2179         self.gitStream.close()
2180         if importProcess.wait() != 0:
2181             die("fast-import failed: %s" % self.gitError.read())
2182         self.gitOutput.close()
2183         self.gitError.close()
2185         return True
2187 class P4Rebase(Command):
2188     def __init__(self):
2189         Command.__init__(self)
2190         self.options = [ ]
2191         self.description = ("Fetches the latest revision from perforce and "
2192                             + "rebases the current work (branch) against it")
2193         self.verbose = False
2195     def run(self, args):
2196         sync = P4Sync()
2197         sync.run([])
2199         return self.rebase()
2201     def rebase(self):
2202         if os.system("git update-index --refresh") != 0:
2203             die("Some files in your working directory are modified and different than what is in your index. You can use git update-index <filename> to bring the index up-to-date or stash away all your changes with git stash.");
2204         if len(read_pipe("git diff-index HEAD --")) > 0:
2205             die("You have uncommited changes. Please commit them before rebasing or stash them away with git stash.");
2207         [upstream, settings] = findUpstreamBranchPoint()
2208         if len(upstream) == 0:
2209             die("Cannot find upstream branchpoint for rebase")
2211         # the branchpoint may be p4/foo~3, so strip off the parent
2212         upstream = re.sub("~[0-9]+$", "", upstream)
2214         print "Rebasing the current branch onto %s" % upstream
2215         oldHead = read_pipe("git rev-parse HEAD").strip()
2216         system("git rebase %s" % upstream)
2217         system("git diff-tree --stat --summary -M %s HEAD" % oldHead)
2218         return True
2220 class P4Clone(P4Sync):
2221     def __init__(self):
2222         P4Sync.__init__(self)
2223         self.description = "Creates a new git repository and imports from Perforce into it"
2224         self.usage = "usage: %prog [options] //depot/path[@revRange]"
2225         self.options += [
2226             optparse.make_option("--destination", dest="cloneDestination",
2227                                  action='store', default=None,
2228                                  help="where to leave result of the clone"),
2229             optparse.make_option("-/", dest="cloneExclude",
2230                                  action="append", type="string",
2231                                  help="exclude depot path"),
2232             optparse.make_option("--bare", dest="cloneBare",
2233                                  action="store_true", default=False),
2234         ]
2235         self.cloneDestination = None
2236         self.needsGit = False
2237         self.cloneBare = False
2239     # This is required for the "append" cloneExclude action
2240     def ensure_value(self, attr, value):
2241         if not hasattr(self, attr) or getattr(self, attr) is None:
2242             setattr(self, attr, value)
2243         return getattr(self, attr)
2245     def defaultDestination(self, args):
2246         ## TODO: use common prefix of args?
2247         depotPath = args[0]
2248         depotDir = re.sub("(@[^@]*)$", "", depotPath)
2249         depotDir = re.sub("(#[^#]*)$", "", depotDir)
2250         depotDir = re.sub(r"\.\.\.$", "", depotDir)
2251         depotDir = re.sub(r"/$", "", depotDir)
2252         return os.path.split(depotDir)[1]
2254     def run(self, args):
2255         if len(args) < 1:
2256             return False
2258         if self.keepRepoPath and not self.cloneDestination:
2259             sys.stderr.write("Must specify destination for --keep-path\n")
2260             sys.exit(1)
2262         depotPaths = args
2264         if not self.cloneDestination and len(depotPaths) > 1:
2265             self.cloneDestination = depotPaths[-1]
2266             depotPaths = depotPaths[:-1]
2268         self.cloneExclude = ["/"+p for p in self.cloneExclude]
2269         for p in depotPaths:
2270             if not p.startswith("//"):
2271                 return False
2273         if not self.cloneDestination:
2274             self.cloneDestination = self.defaultDestination(args)
2276         print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination)
2278         if not os.path.exists(self.cloneDestination):
2279             os.makedirs(self.cloneDestination)
2280         chdir(self.cloneDestination)
2282         init_cmd = [ "git", "init" ]
2283         if self.cloneBare:
2284             init_cmd.append("--bare")
2285         subprocess.check_call(init_cmd)
2287         if not P4Sync.run(self, depotPaths):
2288             return False
2289         if self.branch != "master":
2290             if self.importIntoRemotes:
2291                 masterbranch = "refs/remotes/p4/master"
2292             else:
2293                 masterbranch = "refs/heads/p4/master"
2294             if gitBranchExists(masterbranch):
2295                 system("git branch master %s" % masterbranch)
2296                 if not self.cloneBare:
2297                     system("git checkout -f")
2298             else:
2299                 print "Could not detect main branch. No checkout/master branch created."
2301         return True
2303 class P4Branches(Command):
2304     def __init__(self):
2305         Command.__init__(self)
2306         self.options = [ ]
2307         self.description = ("Shows the git branches that hold imports and their "
2308                             + "corresponding perforce depot paths")
2309         self.verbose = False
2311     def run(self, args):
2312         if originP4BranchesExist():
2313             createOrUpdateBranchesFromOrigin()
2315         cmdline = "git rev-parse --symbolic "
2316         cmdline += " --remotes"
2318         for line in read_pipe_lines(cmdline):
2319             line = line.strip()
2321             if not line.startswith('p4/') or line == "p4/HEAD":
2322                 continue
2323             branch = line
2325             log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch)
2326             settings = extractSettingsGitLog(log)
2328             print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"])
2329         return True
2331 class HelpFormatter(optparse.IndentedHelpFormatter):
2332     def __init__(self):
2333         optparse.IndentedHelpFormatter.__init__(self)
2335     def format_description(self, description):
2336         if description:
2337             return description + "\n"
2338         else:
2339             return ""
2341 def printUsage(commands):
2342     print "usage: %s <command> [options]" % sys.argv[0]
2343     print ""
2344     print "valid commands: %s" % ", ".join(commands)
2345     print ""
2346     print "Try %s <command> --help for command specific help." % sys.argv[0]
2347     print ""
2349 commands = {
2350     "debug" : P4Debug,
2351     "submit" : P4Submit,
2352     "commit" : P4Submit,
2353     "sync" : P4Sync,
2354     "rebase" : P4Rebase,
2355     "clone" : P4Clone,
2356     "rollback" : P4RollBack,
2357     "branches" : P4Branches
2361 def main():
2362     if len(sys.argv[1:]) == 0:
2363         printUsage(commands.keys())
2364         sys.exit(2)
2366     cmd = ""
2367     cmdName = sys.argv[1]
2368     try:
2369         klass = commands[cmdName]
2370         cmd = klass()
2371     except KeyError:
2372         print "unknown command %s" % cmdName
2373         print ""
2374         printUsage(commands.keys())
2375         sys.exit(2)
2377     options = cmd.options
2378     cmd.gitdir = os.environ.get("GIT_DIR", None)
2380     args = sys.argv[2:]
2382     if len(options) > 0:
2383         if cmd.needsGit:
2384             options.append(optparse.make_option("--git-dir", dest="gitdir"))
2386         parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
2387                                        options,
2388                                        description = cmd.description,
2389                                        formatter = HelpFormatter())
2391         (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
2392     global verbose
2393     verbose = cmd.verbose
2394     if cmd.needsGit:
2395         if cmd.gitdir == None:
2396             cmd.gitdir = os.path.abspath(".git")
2397             if not isValidGitDir(cmd.gitdir):
2398                 cmd.gitdir = read_pipe("git rev-parse --git-dir").strip()
2399                 if os.path.exists(cmd.gitdir):
2400                     cdup = read_pipe("git rev-parse --show-cdup").strip()
2401                     if len(cdup) > 0:
2402                         chdir(cdup);
2404         if not isValidGitDir(cmd.gitdir):
2405             if isValidGitDir(cmd.gitdir + "/.git"):
2406                 cmd.gitdir += "/.git"
2407             else:
2408                 die("fatal: cannot locate git repository at %s" % cmd.gitdir)
2410         os.environ["GIT_DIR"] = cmd.gitdir
2412     if not cmd.run(args):
2413         parser.print_help()
2414         sys.exit(2)
2417 if __name__ == '__main__':
2418     main()