contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <hausmann@kde.org>
   6 # Copyright: 2007 Simon Hausmann <hausmann@kde.org>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, shelve
  12 import tempfile, getopt, sha, os.path, time
  13 from sets import Set;
  14
  15 gitdir = os.environ.get("GIT_DIR", "")
  16
  17 def p4CmdList(cmd):
  18     cmd = "p4 -G %s" % cmd
  19     pipe = os.popen(cmd, "rb")
  20
  21     result = []
  22     try:
  23         while True:
  24             entry = marshal.load(pipe)
  25             result.append(entry)
  26     except EOFError:
  27         pass
  28     pipe.close()
  29
  30     return result
  31
  32 def p4Cmd(cmd):
  33     list = p4CmdList(cmd)
  34     result = {}
  35     for entry in list:
  36         result.update(entry)
  37     return result;
  38
  39 def p4Where(depotPath):
  40     if not depotPath.endswith("/"):
  41         depotPath += "/"
  42     output = p4Cmd("where %s..." % depotPath)
  43     clientPath = ""
  44     if "path" in output:
  45         clientPath = output.get("path")
  46     elif "data" in output:
  47         data = output.get("data")
  48         lastSpace = data.rfind(" ")
  49         clientPath = data[lastSpace + 1:]
  50
  51     if clientPath.endswith("..."):
  52         clientPath = clientPath[:-3]
  53     return clientPath
  54
  55 def die(msg):
  56     sys.stderr.write(msg + "\n")
  57     sys.exit(1)
  58
  59 def currentGitBranch():
  60     return os.popen("git-name-rev HEAD").read().split(" ")[1][:-1]
  61
  62 def isValidGitDir(path):
  63     if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
  64         return True;
  65     return False
  66
  67 def system(cmd):
  68     if os.system(cmd) != 0:
  69         die("command failed: %s" % cmd)
  70
  71 def extractLogMessageFromGitCommit(commit):
  72     logMessage = ""
  73     foundTitle = False
  74     for log in os.popen("git-cat-file commit %s" % commit).readlines():
  75        if not foundTitle:
  76            if len(log) == 1:
  77                foundTitle = 1
  78            continue
  79
  80        logMessage += log
  81     return logMessage
  82
  83 def extractDepotPathAndChangeFromGitLog(log):
  84     values = {}
  85     for line in log.split("\n"):
  86         line = line.strip()
  87         if line.startswith("[git-p4:") and line.endswith("]"):
  88             line = line[8:-1].strip()
  89             for assignment in line.split(":"):
  90                 variable = assignment.strip()
  91                 value = ""
  92                 equalPos = assignment.find("=")
  93                 if equalPos != -1:
  94                     variable = assignment[:equalPos].strip()
  95                     value = assignment[equalPos + 1:].strip()
  96                     if value.startswith("\"") and value.endswith("\""):
  97                         value = value[1:-1]
  98                 values[variable] = value
  99
 100     return values.get("depot-path"), values.get("change")
 101
 102 def gitBranchExists(branch):
 103     if os.system("git-rev-parse %s 2>/dev/null >/dev/null" % branch) == 0:
 104         return True
 105     return False
 106
 107 class Command:
 108     def __init__(self):
 109         self.usage = "usage: %prog [options]"
 110
 111 class P4Debug(Command):
 112     def __init__(self):
 113         Command.__init__(self)
 114         self.options = [
 115         ]
 116         self.description = "A tool to debug the output of p4 -G."
 117
 118     def run(self, args):
 119         for output in p4CmdList(" ".join(args)):
 120             print output
 121         return True
 122
 123 class P4CleanTags(Command):
 124     def __init__(self):
 125         Command.__init__(self)
 126         self.options = [
 127 #                optparse.make_option("--branch", dest="branch", default="refs/heads/master")
 128         ]
 129         self.description = "A tool to remove stale unused tags from incremental perforce imports."
 130     def run(self, args):
 131         branch = currentGitBranch()
 132         print "Cleaning out stale p4 import tags..."
 133         sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
 134         output = sout.read()
 135         try:
 136             tagIdx = output.index(" tags/p4/")
 137         except:
 138             print "Cannot find any p4/* tag. Nothing to do."
 139             sys.exit(0)
 140
 141         try:
 142             caretIdx = output.index("^")
 143         except:
 144             caretIdx = len(output) - 1
 145         rev = int(output[tagIdx + 9 : caretIdx])
 146
 147         allTags = os.popen("git tag -l p4/").readlines()
 148         for i in range(len(allTags)):
 149             allTags[i] = int(allTags[i][3:-1])
 150
 151         allTags.sort()
 152
 153         allTags.remove(rev)
 154
 155         for rev in allTags:
 156             print os.popen("git tag -d p4/%s" % rev).read()
 157
 158         print "%s tags removed." % len(allTags)
 159         return True
 160
 161 class P4Sync(Command):
 162     def __init__(self):
 163         Command.__init__(self)
 164         self.options = [
 165                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
 166                 optparse.make_option("--origin", dest="origin"),
 167                 optparse.make_option("--reset", action="store_true", dest="reset"),
 168                 optparse.make_option("--master", dest="master"),
 169                 optparse.make_option("--log-substitutions", dest="substFile"),
 170                 optparse.make_option("--noninteractive", action="store_false"),
 171                 optparse.make_option("--dry-run", action="store_true"),
 172                 optparse.make_option("--apply-as-patch", action="store_true", dest="applyAsPatch")
 173         ]
 174         self.description = "Submit changes from git to the perforce depot."
 175         self.firstTime = True
 176         self.reset = False
 177         self.interactive = True
 178         self.dryRun = False
 179         self.substFile = ""
 180         self.firstTime = True
 181         self.origin = ""
 182         self.master = ""
 183         self.applyAsPatch = True
 184
 185         self.logSubstitutions = {}
 186         self.logSubstitutions["<enter description here>"] = "%log%"
 187         self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 188
 189     def check(self):
 190         if len(p4CmdList("opened ...")) > 0:
 191             die("You have files opened with perforce! Close them before starting the sync.")
 192
 193     def start(self):
 194         if len(self.config) > 0 and not self.reset:
 195             die("Cannot start sync. Previous sync config found at %s" % self.configFile)
 196
 197         commits = []
 198         for line in os.popen("git-rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
 199             commits.append(line[:-1])
 200         commits.reverse()
 201
 202         self.config["commits"] = commits
 203
 204         if not self.applyAsPatch:
 205             print "Creating temporary p4-sync branch from %s ..." % self.origin
 206             system("git checkout -f -b p4-sync %s" % self.origin)
 207
 208     def prepareLogMessage(self, template, message):
 209         result = ""
 210
 211         for line in template.split("\n"):
 212             if line.startswith("#"):
 213                 result += line + "\n"
 214                 continue
 215
 216             substituted = False
 217             for key in self.logSubstitutions.keys():
 218                 if line.find(key) != -1:
 219                     value = self.logSubstitutions[key]
 220                     value = value.replace("%log%", message)
 221                     if value != "@remove@":
 222                         result += line.replace(key, value) + "\n"
 223                     substituted = True
 224                     break
 225
 226             if not substituted:
 227                 result += line + "\n"
 228
 229         return result
 230
 231     def apply(self, id):
 232         print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read())
 233         diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
 234         filesToAdd = set()
 235         filesToDelete = set()
 236         for line in diff:
 237             modifier = line[0]
 238             path = line[1:].strip()
 239             if modifier == "M":
 240                 system("p4 edit %s" % path)
 241             elif modifier == "A":
 242                 filesToAdd.add(path)
 243                 if path in filesToDelete:
 244                     filesToDelete.remove(path)
 245             elif modifier == "D":
 246                 filesToDelete.add(path)
 247                 if path in filesToAdd:
 248                     filesToAdd.remove(path)
 249             else:
 250                 die("unknown modifier %s for %s" % (modifier, path))
 251
 252         if self.applyAsPatch:
 253             system("git-diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\" | patch -p1" % (id, id))
 254         else:
 255             system("git-diff-files --name-only -z | git-update-index --remove -z --stdin")
 256             system("git cherry-pick --no-commit \"%s\"" % id)
 257
 258         for f in filesToAdd:
 259             system("p4 add %s" % f)
 260         for f in filesToDelete:
 261             system("p4 revert %s" % f)
 262             system("p4 delete %s" % f)
 263
 264         logMessage = extractLogMessageFromGitCommit(id)
 265         logMessage = logMessage.replace("\n", "\n\t")
 266         logMessage = logMessage[:-1]
 267
 268         template = os.popen("p4 change -o").read()
 269
 270         if self.interactive:
 271             submitTemplate = self.prepareLogMessage(template, logMessage)
 272             diff = os.popen("p4 diff -du ...").read()
 273
 274             for newFile in filesToAdd:
 275                 diff += "==== new file ====\n"
 276                 diff += "--- /dev/null\n"
 277                 diff += "+++ %s\n" % newFile
 278                 f = open(newFile, "r")
 279                 for line in f.readlines():
 280                     diff += "+" + line
 281                 f.close()
 282
 283             separatorLine = "######## everything below this line is just the diff #######\n"
 284
 285             response = "e"
 286             firstIteration = True
 287             while response == "e":
 288                 if not firstIteration:
 289                     response = raw_input("Do you want to submit this change (y/e/n)? ")
 290                 firstIteration = False
 291                 if response == "e":
 292                     [handle, fileName] = tempfile.mkstemp()
 293                     tmpFile = os.fdopen(handle, "w+")
 294                     tmpFile.write(submitTemplate + separatorLine + diff)
 295                     tmpFile.close()
 296                     editor = os.environ.get("EDITOR", "vi")
 297                     system(editor + " " + fileName)
 298                     tmpFile = open(fileName, "r")
 299                     message = tmpFile.read()
 300                     tmpFile.close()
 301                     os.remove(fileName)
 302                     submitTemplate = message[:message.index(separatorLine)]
 303
 304             if response == "y" or response == "yes":
 305                if self.dryRun:
 306                    print submitTemplate
 307                    raw_input("Press return to continue...")
 308                else:
 309                     pipe = os.popen("p4 submit -i", "w")
 310                     pipe.write(submitTemplate)
 311                     pipe.close()
 312             else:
 313                 print "Not submitting!"
 314                 self.interactive = False
 315         else:
 316             fileName = "submit.txt"
 317             file = open(fileName, "w+")
 318             file.write(self.prepareLogMessage(template, logMessage))
 319             file.close()
 320             print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)
 321
 322     def run(self, args):
 323         global gitdir
 324         # make gitdir absolute so we can cd out into the perforce checkout
 325         gitdir = os.path.abspath(gitdir)
 326         os.environ["GIT_DIR"] = gitdir
 327         depotPath = ""
 328         if gitBranchExists("p4"):
 329             [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4"))
 330         if len(depotPath) == 0 and gitBranchExists("origin"):
 331             [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin"))
 332
 333         if len(depotPath) == 0:
 334             print "Internal error: cannot locate perforce depot path from existing branches"
 335             sys.exit(128)
 336
 337         clientPath = p4Where(depotPath)
 338
 339         if len(clientPath) == 0:
 340             print "Error: Cannot locate perforce checkout of %s in client view" % depotPath
 341             sys.exit(128)
 342
 343         print "Perforce checkout for depot path %s located at %s" % (depotPath, clientPath)
 344         os.chdir(clientPath)
 345         response = raw_input("Do you want to sync %s with p4 sync? (y/n)" % clientPath)
 346         if response == "y" or response == "yes":
 347             system("p4 sync ...")
 348
 349         if len(self.origin) == 0:
 350             if gitBranchExists("p4"):
 351                 self.origin = "p4"
 352             else:
 353                 self.origin = "origin"
 354
 355         if self.reset:
 356             self.firstTime = True
 357
 358         if len(self.substFile) > 0:
 359             for line in open(self.substFile, "r").readlines():
 360                 tokens = line[:-1].split("=")
 361                 self.logSubstitutions[tokens[0]] = tokens[1]
 362
 363         if len(self.master) == 0:
 364             self.master = currentGitBranch()
 365             if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
 366                 die("Detecting current git branch failed!")
 367
 368         self.check()
 369         self.configFile = gitdir + "/p4-git-sync.cfg"
 370         self.config = shelve.open(self.configFile, writeback=True)
 371
 372         if self.firstTime:
 373             self.start()
 374
 375         commits = self.config.get("commits", [])
 376
 377         while len(commits) > 0:
 378             self.firstTime = False
 379             commit = commits[0]
 380             commits = commits[1:]
 381             self.config["commits"] = commits
 382             self.apply(commit)
 383             if not self.interactive:
 384                 break
 385
 386         self.config.close()
 387
 388         if len(commits) == 0:
 389             if self.firstTime:
 390                 print "No changes found to apply between %s and current HEAD" % self.origin
 391             else:
 392                 print "All changes applied!"
 393                 if not self.applyAsPatch:
 394                     print "Deleting temporary p4-sync branch and going back to %s" % self.master
 395                     system("git checkout %s" % self.master)
 396                     system("git branch -D p4-sync")
 397                     print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..."
 398                     system("p4 edit ... >/dev/null")
 399                     system("p4 revert ... >/dev/null")
 400             os.remove(self.configFile)
 401
 402         return True
 403
 404 class GitSync(Command):
 405     def __init__(self):
 406         Command.__init__(self)
 407         self.options = [
 408                 optparse.make_option("--branch", dest="branch"),
 409                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 410                 optparse.make_option("--changesfile", dest="changesFile"),
 411                 optparse.make_option("--silent", dest="silent", action="store_true"),
 412                 optparse.make_option("--known-branches", dest="knownBranches"),
 413                 optparse.make_option("--cache", dest="doCache", action="store_true"),
 414                 optparse.make_option("--command-cache", dest="commandCache", action="store_true")
 415         ]
 416         self.description = """Imports from Perforce into a git repository.\n
 417     example:
 418     //depot/my/project/ -- to import the current head
 419     //depot/my/project/@all -- to import everything
 420     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 421
 422     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 423
 424         self.usage += " //depot/path[@revRange]"
 425
 426         self.dataCache = False
 427         self.commandCache = False
 428         self.silent = False
 429         self.knownBranches = Set()
 430         self.createdBranches = Set()
 431         self.committedChanges = Set()
 432         self.branch = ""
 433         self.detectBranches = False
 434         self.changesFile = ""
 435
 436     def p4File(self, depotPath):
 437         return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 438
 439     def extractFilesFromCommit(self, commit):
 440         files = []
 441         fnum = 0
 442         while commit.has_key("depotFile%s" % fnum):
 443             path =  commit["depotFile%s" % fnum]
 444             if not path.startswith(self.globalPrefix):
 445     #            if not self.silent:
 446     #                print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.globalPrefix, change)
 447                 fnum = fnum + 1
 448                 continue
 449
 450             file = {}
 451             file["path"] = path
 452             file["rev"] = commit["rev%s" % fnum]
 453             file["action"] = commit["action%s" % fnum]
 454             file["type"] = commit["type%s" % fnum]
 455             files.append(file)
 456             fnum = fnum + 1
 457         return files
 458
 459     def isSubPathOf(self, first, second):
 460         if not first.startswith(second):
 461             return False
 462         if first == second:
 463             return True
 464         return first[len(second)] == "/"
 465
 466     def branchesForCommit(self, files):
 467         branches = Set()
 468
 469         for file in files:
 470             relativePath = file["path"][len(self.globalPrefix):]
 471             # strip off the filename
 472             relativePath = relativePath[0:relativePath.rfind("/")]
 473
 474     #        if len(branches) == 0:
 475     #            branches.add(relativePath)
 476     #            knownBranches.add(relativePath)
 477     #            continue
 478
 479             ###### this needs more testing :)
 480             knownBranch = False
 481             for branch in branches:
 482                 if relativePath == branch:
 483                     knownBranch = True
 484                     break
 485     #            if relativePath.startswith(branch):
 486                 if self.isSubPathOf(relativePath, branch):
 487                     knownBranch = True
 488                     break
 489     #            if branch.startswith(relativePath):
 490                 if self.isSubPathOf(branch, relativePath):
 491                     branches.remove(branch)
 492                     break
 493
 494             if knownBranch:
 495                 continue
 496
 497             for branch in knownBranches:
 498                 #if relativePath.startswith(branch):
 499                 if self.isSubPathOf(relativePath, branch):
 500                     if len(branches) == 0:
 501                         relativePath = branch
 502                     else:
 503                         knownBranch = True
 504                     break
 505
 506             if knownBranch:
 507                 continue
 508
 509             branches.add(relativePath)
 510             self.knownBranches.add(relativePath)
 511
 512         return branches
 513
 514     def findBranchParent(self, branchPrefix, files):
 515         for file in files:
 516             path = file["path"]
 517             if not path.startswith(branchPrefix):
 518                 continue
 519             action = file["action"]
 520             if action != "integrate" and action != "branch":
 521                 continue
 522             rev = file["rev"]
 523             depotPath = path + "#" + rev
 524
 525             log = p4CmdList("filelog \"%s\"" % depotPath)
 526             if len(log) != 1:
 527                 print "eek! I got confused by the filelog of %s" % depotPath
 528                 sys.exit(1);
 529
 530             log = log[0]
 531             if log["action0"] != action:
 532                 print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 533                 sys.exit(1);
 534
 535             branchAction = log["how0,0"]
 536     #        if branchAction == "branch into" or branchAction == "ignored":
 537     #            continue # ignore for branching
 538
 539             if not branchAction.endswith(" from"):
 540                 continue # ignore for branching
 541     #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 542     #            sys.exit(1);
 543
 544             source = log["file0,0"]
 545             if source.startswith(branchPrefix):
 546                 continue
 547
 548             lastSourceRev = log["erev0,0"]
 549
 550             sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 551             if len(sourceLog) != 1:
 552                 print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 553                 sys.exit(1);
 554             sourceLog = sourceLog[0]
 555
 556             relPath = source[len(self.globalPrefix):]
 557             # strip off the filename
 558             relPath = relPath[0:relPath.rfind("/")]
 559
 560             for branch in self.knownBranches:
 561                 if self.isSubPathOf(relPath, branch):
 562     #                print "determined parent branch branch %s due to change in file %s" % (branch, source)
 563                     return branch
 564     #            else:
 565     #                print "%s is not a subpath of branch %s" % (relPath, branch)
 566
 567         return ""
 568
 569     def commit(self, details, files, branch, branchPrefix, parent = "", merged = ""):
 570         epoch = details["time"]
 571         author = details["user"]
 572
 573         self.gitStream.write("commit %s\n" % branch)
 574     #    gitStream.write("mark :%s\n" % details["change"])
 575         self.committedChanges.add(int(details["change"]))
 576         committer = ""
 577         if author in self.users:
 578             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 579         else:
 580             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 581
 582         self.gitStream.write("committer %s\n" % committer)
 583
 584         self.gitStream.write("data <<EOT\n")
 585         self.gitStream.write(details["desc"])
 586         self.gitStream.write("\n[git-p4: depot-path = \"%s\": change = %s]\n" % (branchPrefix, details["change"]))
 587         self.gitStream.write("EOT\n\n")
 588
 589         if len(parent) > 0:
 590             self.gitStream.write("from %s\n" % parent)
 591
 592         if len(merged) > 0:
 593             self.gitStream.write("merge %s\n" % merged)
 594
 595         for file in files:
 596             path = file["path"]
 597             if not path.startswith(branchPrefix):
 598     #            if not silent:
 599     #                print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 600                 continue
 601             rev = file["rev"]
 602             depotPath = path + "#" + rev
 603             relPath = path[len(branchPrefix):]
 604             action = file["action"]
 605
 606             if file["type"] == "apple":
 607                 print "\nfile %s is a strange apple file that forks. Ignoring!" % path
 608                 continue
 609
 610             if action == "delete":
 611                 self.gitStream.write("D %s\n" % relPath)
 612             else:
 613                 mode = 644
 614                 if file["type"].startswith("x"):
 615                     mode = 755
 616
 617                 data = self.p4File(depotPath)
 618
 619                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 620                 self.gitStream.write("data %s\n" % len(data))
 621                 self.gitStream.write(data)
 622                 self.gitStream.write("\n")
 623
 624         self.gitStream.write("\n")
 625
 626         self.lastChange = int(details["change"])
 627
 628     def extractFilesInCommitToBranch(self, files, branchPrefix):
 629         newFiles = []
 630
 631         for file in files:
 632             path = file["path"]
 633             if path.startswith(branchPrefix):
 634                 newFiles.append(file)
 635
 636         return newFiles
 637
 638     def findBranchSourceHeuristic(self, files, branch, branchPrefix):
 639         for file in files:
 640             action = file["action"]
 641             if action != "integrate" and action != "branch":
 642                 continue
 643             path = file["path"]
 644             rev = file["rev"]
 645             depotPath = path + "#" + rev
 646
 647             log = p4CmdList("filelog \"%s\"" % depotPath)
 648             if len(log) != 1:
 649                 print "eek! I got confused by the filelog of %s" % depotPath
 650                 sys.exit(1);
 651
 652             log = log[0]
 653             if log["action0"] != action:
 654                 print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 655                 sys.exit(1);
 656
 657             branchAction = log["how0,0"]
 658
 659             if not branchAction.endswith(" from"):
 660                 continue # ignore for branching
 661     #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 662     #            sys.exit(1);
 663
 664             source = log["file0,0"]
 665             if source.startswith(branchPrefix):
 666                 continue
 667
 668             lastSourceRev = log["erev0,0"]
 669
 670             sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 671             if len(sourceLog) != 1:
 672                 print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 673                 sys.exit(1);
 674             sourceLog = sourceLog[0]
 675
 676             relPath = source[len(self.globalPrefix):]
 677             # strip off the filename
 678             relPath = relPath[0:relPath.rfind("/")]
 679
 680             for candidate in self.knownBranches:
 681                 if self.isSubPathOf(relPath, candidate) and candidate != branch:
 682                     return candidate
 683
 684         return ""
 685
 686     def changeIsBranchMerge(self, sourceBranch, destinationBranch, change):
 687         sourceFiles = {}
 688         for file in p4CmdList("files %s...@%s" % (self.globalPrefix + sourceBranch + "/", change)):
 689             if file["action"] == "delete":
 690                 continue
 691             sourceFiles[file["depotFile"]] = file
 692
 693         destinationFiles = {}
 694         for file in p4CmdList("files %s...@%s" % (self.globalPrefix + destinationBranch + "/", change)):
 695             destinationFiles[file["depotFile"]] = file
 696
 697         for fileName in sourceFiles.keys():
 698             integrations = []
 699             deleted = False
 700             integrationCount = 0
 701             for integration in p4CmdList("integrated \"%s\"" % fileName):
 702                 toFile = integration["fromFile"] # yes, it's true, it's fromFile
 703                 if not toFile in destinationFiles:
 704                     continue
 705                 destFile = destinationFiles[toFile]
 706                 if destFile["action"] == "delete":
 707     #                print "file %s has been deleted in %s" % (fileName, toFile)
 708                     deleted = True
 709                     break
 710                 integrationCount += 1
 711                 if integration["how"] == "branch from":
 712                     continue
 713
 714                 if int(integration["change"]) == change:
 715                     integrations.append(integration)
 716                     continue
 717                 if int(integration["change"]) > change:
 718                     continue
 719
 720                 destRev = int(destFile["rev"])
 721
 722                 startRev = integration["startFromRev"][1:]
 723                 if startRev == "none":
 724                     startRev = 0
 725                 else:
 726                     startRev = int(startRev)
 727
 728                 endRev = integration["endFromRev"][1:]
 729                 if endRev == "none":
 730                     endRev = 0
 731                 else:
 732                     endRev = int(endRev)
 733
 734                 initialBranch = (destRev == 1 and integration["how"] != "branch into")
 735                 inRange = (destRev >= startRev and destRev <= endRev)
 736                 newer = (destRev > startRev and destRev > endRev)
 737
 738                 if initialBranch or inRange or newer:
 739                     integrations.append(integration)
 740
 741             if deleted:
 742                 continue
 743
 744             if len(integrations) == 0 and integrationCount > 1:
 745                 print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch)
 746                 return False
 747
 748         return True
 749
 750     def getUserMap(self):
 751         self.users = {}
 752
 753         for output in p4CmdList("users"):
 754             if not output.has_key("User"):
 755                 continue
 756             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 757
 758     def run(self, args):
 759         self.globalPrefix = ""
 760         self.changeRange = ""
 761         self.initialParent = ""
 762         self.tagLastChange = True
 763
 764         if len(self.branch) == 0:
 765             self.branch = "p4"
 766
 767         if len(args) == 0:
 768             if not gitBranchExists(self.branch) and gitBranchExists("origin"):
 769                 if not self.silent:
 770                     print "Creating %s branch in git repository based on origin" % self.branch
 771                 system("git branch %s origin" % self.branch)
 772
 773             [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch))
 774             if len(self.previousDepotPath) > 0 and len(p4Change) > 0:
 775                 p4Change = int(p4Change) + 1
 776                 self.globalPrefix = self.previousDepotPath
 777                 self.changeRange = "@%s,#head" % p4Change
 778                 self.initialParent = self.branch
 779                 self.tagLastChange = False
 780                 if not self.silent:
 781                     print "Performing incremental import into %s git branch" % self.branch
 782
 783         self.branch = "refs/heads/" + self.branch
 784
 785         if len(self.globalPrefix) == 0:
 786             self.globalPrefix = self.previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read()
 787
 788         if len(self.globalPrefix) != 0:
 789             self.globalPrefix = self.globalPrefix[:-1]
 790
 791         if len(args) == 0 and len(self.globalPrefix) != 0:
 792             if not self.silent:
 793                 print "Depot path: %s" % self.globalPrefix
 794         elif len(args) != 1:
 795             return False
 796         else:
 797             if len(self.globalPrefix) != 0 and self.globalPrefix != args[0]:
 798                 print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.globalPrefix, args[0])
 799                 sys.exit(1)
 800             self.globalPrefix = args[0]
 801
 802         self.revision = ""
 803         self.users = {}
 804         self.lastChange = 0
 805         self.initialTag = ""
 806
 807         if self.globalPrefix.find("@") != -1:
 808             atIdx = self.globalPrefix.index("@")
 809             self.changeRange = self.globalPrefix[atIdx:]
 810             if self.changeRange == "@all":
 811                 self.changeRange = ""
 812             elif self.changeRange.find(",") == -1:
 813                 self.revision = self.changeRange
 814                 self.changeRange = ""
 815             self.globalPrefix = self.globalPrefix[0:atIdx]
 816         elif self.globalPrefix.find("#") != -1:
 817             hashIdx = self.globalPrefix.index("#")
 818             self.revision = self.globalPrefix[hashIdx:]
 819             self.globalPrefix = self.globalPrefix[0:hashIdx]
 820         elif len(self.previousDepotPath) == 0:
 821             self.revision = "#head"
 822
 823         if self.globalPrefix.endswith("..."):
 824             self.globalPrefix = self.globalPrefix[:-3]
 825
 826         if not self.globalPrefix.endswith("/"):
 827             self.globalPrefix += "/"
 828
 829         self.getUserMap()
 830
 831         if len(self.changeRange) == 0:
 832             try:
 833                 sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % self.branch)
 834                 output = sout.read()
 835                 if output.endswith("\n"):
 836                     output = output[:-1]
 837                 tagIdx = output.index(" tags/p4/")
 838                 caretIdx = output.find("^")
 839                 endPos = len(output)
 840                 if caretIdx != -1:
 841                     endPos = caretIdx
 842                 self.rev = int(output[tagIdx + 9 : endPos]) + 1
 843                 self.changeRange = "@%s,#head" % self.rev
 844                 self.initialParent = os.popen("git-rev-parse %s" % self.branch).read()[:-1]
 845                 self.initialTag = "p4/%s" % (int(self.rev) - 1)
 846             except:
 847                 pass
 848
 849         self.tz = - time.timezone / 36
 850         tzsign = ("%s" % self.tz)[0]
 851         if tzsign != '+' and tzsign != '-':
 852             self.tz = "+" + ("%s" % self.tz)
 853
 854         self.gitOutput, self.gitStream, self.gitError = popen2.popen3("git-fast-import")
 855
 856         if len(self.revision) > 0:
 857             print "Doing initial import of %s from revision %s" % (self.globalPrefix, self.revision)
 858
 859             details = { "user" : "git perforce import user", "time" : int(time.time()) }
 860             details["desc"] = "Initial import of %s from the state at revision %s" % (self.globalPrefix, self.revision)
 861             details["change"] = self.revision
 862             newestRevision = 0
 863
 864             fileCnt = 0
 865             for info in p4CmdList("files %s...%s" % (self.globalPrefix, self.revision)):
 866                 change = int(info["change"])
 867                 if change > newestRevision:
 868                     newestRevision = change
 869
 870                 if info["action"] == "delete":
 871                     fileCnt = fileCnt + 1
 872                     continue
 873
 874                 for prop in [ "depotFile", "rev", "action", "type" ]:
 875                     details["%s%s" % (prop, fileCnt)] = info[prop]
 876
 877                 fileCnt = fileCnt + 1
 878
 879             details["change"] = newestRevision
 880
 881             try:
 882                 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.globalPrefix)
 883             except IOError:
 884                 print self.gitError.read()
 885
 886         else:
 887             changes = []
 888
 889             if len(self.changesFile) > 0:
 890                 output = open(self.changesFile).readlines()
 891                 changeSet = Set()
 892                 for line in output:
 893                     changeSet.add(int(line))
 894
 895                 for change in changeSet:
 896                     changes.append(change)
 897
 898                 changes.sort()
 899             else:
 900                 output = os.popen("p4 changes %s...%s" % (self.globalPrefix, self.changeRange)).readlines()
 901
 902                 for line in output:
 903                     changeNum = line.split(" ")[1]
 904                     changes.append(changeNum)
 905
 906                 changes.reverse()
 907
 908             if len(changes) == 0:
 909                 if not self.silent:
 910                     print "no changes to import!"
 911                 sys.exit(1)
 912
 913             cnt = 1
 914             for change in changes:
 915                 description = p4Cmd("describe %s" % change)
 916
 917                 if not self.silent:
 918                     sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
 919                     sys.stdout.flush()
 920                 cnt = cnt + 1
 921
 922                 try:
 923                     files = self.extractFilesFromCommit(description)
 924                     if self.detectBranches:
 925                         for branch in self.branchesForCommit(files):
 926                             self.knownBranches.add(branch)
 927                             branchPrefix = self.globalPrefix + branch + "/"
 928
 929                             filesForCommit = self.extractFilesInCommitToBranch(files, branchPrefix)
 930
 931                             merged = ""
 932                             parent = ""
 933                             ########### remove cnt!!!
 934                             if branch not in self.createdBranches and cnt > 2:
 935                                 self.createdBranches.add(branch)
 936                                 parent = self.findBranchParent(branchPrefix, files)
 937                                 if parent == branch:
 938                                     parent = ""
 939             #                    elif len(parent) > 0:
 940             #                        print "%s branched off of %s" % (branch, parent)
 941
 942                             if len(parent) == 0:
 943                                 merged = self.findBranchSourceHeuristic(filesForCommit, branch, branchPrefix)
 944                                 if len(merged) > 0:
 945                                     print "change %s could be a merge from %s into %s" % (description["change"], merged, branch)
 946                                     if not self.changeIsBranchMerge(merged, branch, int(description["change"])):
 947                                         merged = ""
 948
 949                             branch = "refs/heads/" + branch
 950                             if len(parent) > 0:
 951                                 parent = "refs/heads/" + parent
 952                             if len(merged) > 0:
 953                                 merged = "refs/heads/" + merged
 954                             self.commit(description, files, branch, branchPrefix, parent, merged)
 955                     else:
 956                         self.commit(description, files, self.branch, self.globalPrefix, self.initialParent)
 957                         self.initialParent = ""
 958                 except IOError:
 959                     print self.gitError.read()
 960                     sys.exit(1)
 961
 962         if not self.silent:
 963             print ""
 964
 965         if self.tagLastChange:
 966             self.gitStream.write("reset refs/tags/p4/%s\n" % self.lastChange)
 967             self.gitStream.write("from %s\n\n" % self.branch);
 968
 969
 970         self.gitStream.close()
 971         self.gitOutput.close()
 972         self.gitError.close()
 973
 974         os.popen("git-repo-config p4.depotpath %s" % self.globalPrefix).read()
 975         if len(self.initialTag) > 0:
 976             os.popen("git tag -d %s" % self.initialTag).read()
 977
 978         return True
 979
 980 class HelpFormatter(optparse.IndentedHelpFormatter):
 981     def __init__(self):
 982         optparse.IndentedHelpFormatter.__init__(self)
 983
 984     def format_description(self, description):
 985         if description:
 986             return description + "\n"
 987         else:
 988             return ""
 989
 990 def printUsage(commands):
 991     print "usage: %s <command> [options]" % sys.argv[0]
 992     print ""
 993     print "valid commands: %s" % ", ".join(commands)
 994     print ""
 995     print "Try %s <command> --help for command specific help." % sys.argv[0]
 996     print ""
 997
 998 commands = {
 999     "debug" : P4Debug(),
1000     "clean-tags" : P4CleanTags(),
1001     "submit" : P4Sync(),
1002     "sync" : GitSync()
1003 }
1004
1005 if len(sys.argv[1:]) == 0:
1006     printUsage(commands.keys())
1007     sys.exit(2)
1008
1009 cmd = ""
1010 cmdName = sys.argv[1]
1011 try:
1012     cmd = commands[cmdName]
1013 except KeyError:
1014     print "unknown command %s" % cmdName
1015     print ""
1016     printUsage(commands.keys())
1017     sys.exit(2)
1018
1019 options = cmd.options
1020 cmd.gitdir = gitdir
1021 options.append(optparse.make_option("--git-dir", dest="gitdir"))
1022
1023 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1024                                options,
1025                                description = cmd.description,
1026                                formatter = HelpFormatter())
1027
1028 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1029
1030 gitdir = cmd.gitdir
1031 if len(gitdir) == 0:
1032     gitdir = ".git"
1033     if not isValidGitDir(gitdir):
1034         cdup = os.popen("git-rev-parse --show-cdup").read()[:-1]
1035         if isValidGitDir(cdup + "/" + gitdir):
1036             os.chdir(cdup)
1037
1038 if not isValidGitDir(gitdir):
1039     if isValidGitDir(gitdir + "/.git"):
1040         gitdir += "/.git"
1041     else:
1042         die("fatal: cannot locate git repository at %s" % gitdir)
1043
1044 os.environ["GIT_DIR"] = gitdir
1045
1046 if not cmd.run(args):
1047     parser.print_help()
1048