contrib/fast-import/git-p4

   1 #!/usr/bin/env python
   2 #
   3 # git-p4.py -- A tool for bidirectional operation between a Perforce depot and git.
   4 #
   5 # Author: Simon Hausmann <hausmann@kde.org>
   6 # Copyright: 2007 Simon Hausmann <hausmann@kde.org>
   7 #            2007 Trolltech ASA
   8 # License: MIT <http://www.opensource.org/licenses/mit-license.php>
   9 #
  10
  11 import optparse, sys, os, marshal, popen2, shelve
  12 import tempfile, getopt, sha, os.path, time
  13 from sets import Set;
  14
  15 gitdir = os.environ.get("GIT_DIR", "")
  16
  17 def p4CmdList(cmd):
  18     cmd = "p4 -G %s" % cmd
  19     pipe = os.popen(cmd, "rb")
  20
  21     result = []
  22     try:
  23         while True:
  24             entry = marshal.load(pipe)
  25             result.append(entry)
  26     except EOFError:
  27         pass
  28     pipe.close()
  29
  30     return result
  31
  32 def p4Cmd(cmd):
  33     list = p4CmdList(cmd)
  34     result = {}
  35     for entry in list:
  36         result.update(entry)
  37     return result;
  38
  39 def die(msg):
  40     sys.stderr.write(msg + "\n")
  41     sys.exit(1)
  42
  43 def currentGitBranch():
  44     return os.popen("git-name-rev HEAD").read().split(" ")[1][:-1]
  45
  46 def isValidGitDir(path):
  47     if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"):
  48         return True;
  49     return False
  50
  51 def system(cmd):
  52     if os.system(cmd) != 0:
  53         die("command failed: %s" % cmd)
  54
  55 def extractLogMessageFromGitCommit(commit):
  56     logMessage = ""
  57     foundTitle = False
  58     for log in os.popen("git-cat-file commit %s" % commit).readlines():
  59        if not foundTitle:
  60            if len(log) == 1:
  61                foundTitle = 1
  62            continue
  63
  64        logMessage += log
  65     return logMessage
  66
  67 def extractDepotPathAndChangeFromGitLog(log):
  68     values = {}
  69     for line in log.split("\n"):
  70         line = line.strip()
  71         if line.startswith("[git-p4:") and line.endswith("]"):
  72             line = line[8:-1].strip()
  73             for assignment in line.split(":"):
  74                 variable = assignment.strip()
  75                 value = ""
  76                 equalPos = assignment.find("=")
  77                 if equalPos != -1:
  78                     variable = assignment[:equalPos].strip()
  79                     value = assignment[equalPos + 1:].strip()
  80                     if value.startswith("\"") and value.endswith("\""):
  81                         value = value[1:-1]
  82                 values[variable] = value
  83
  84     return values.get("depot-path"), values.get("change")
  85
  86 def gitBranchExists(branch):
  87     if os.system("git-rev-parse %s 2>/dev/null >/dev/null" % branch) == 0:
  88         return True
  89     return False
  90
  91 class Command:
  92     def __init__(self):
  93         self.usage = "usage: %prog [options]"
  94
  95 class P4Debug(Command):
  96     def __init__(self):
  97         Command.__init__(self)
  98         self.options = [
  99         ]
 100         self.description = "A tool to debug the output of p4 -G."
 101
 102     def run(self, args):
 103         for output in p4CmdList(" ".join(args)):
 104             print output
 105         return True
 106
 107 class P4CleanTags(Command):
 108     def __init__(self):
 109         Command.__init__(self)
 110         self.options = [
 111 #                optparse.make_option("--branch", dest="branch", default="refs/heads/master")
 112         ]
 113         self.description = "A tool to remove stale unused tags from incremental perforce imports."
 114     def run(self, args):
 115         branch = currentGitBranch()
 116         print "Cleaning out stale p4 import tags..."
 117         sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % branch)
 118         output = sout.read()
 119         try:
 120             tagIdx = output.index(" tags/p4/")
 121         except:
 122             print "Cannot find any p4/* tag. Nothing to do."
 123             sys.exit(0)
 124
 125         try:
 126             caretIdx = output.index("^")
 127         except:
 128             caretIdx = len(output) - 1
 129         rev = int(output[tagIdx + 9 : caretIdx])
 130
 131         allTags = os.popen("git tag -l p4/").readlines()
 132         for i in range(len(allTags)):
 133             allTags[i] = int(allTags[i][3:-1])
 134
 135         allTags.sort()
 136
 137         allTags.remove(rev)
 138
 139         for rev in allTags:
 140             print os.popen("git tag -d p4/%s" % rev).read()
 141
 142         print "%s tags removed." % len(allTags)
 143         return True
 144
 145 class P4Sync(Command):
 146     def __init__(self):
 147         Command.__init__(self)
 148         self.options = [
 149                 optparse.make_option("--continue", action="store_false", dest="firstTime"),
 150                 optparse.make_option("--origin", dest="origin"),
 151                 optparse.make_option("--reset", action="store_true", dest="reset"),
 152                 optparse.make_option("--master", dest="master"),
 153                 optparse.make_option("--log-substitutions", dest="substFile"),
 154                 optparse.make_option("--noninteractive", action="store_false"),
 155                 optparse.make_option("--dry-run", action="store_true"),
 156                 optparse.make_option("--apply-as-patch", action="store_true", dest="applyAsPatch")
 157         ]
 158         self.description = "Submit changes from git to the perforce depot."
 159         self.firstTime = True
 160         self.reset = False
 161         self.interactive = True
 162         self.dryRun = False
 163         self.substFile = ""
 164         self.firstTime = True
 165         self.origin = ""
 166         self.master = ""
 167         self.applyAsPatch = True
 168
 169         self.logSubstitutions = {}
 170         self.logSubstitutions["<enter description here>"] = "%log%"
 171         self.logSubstitutions["\tDetails:"] = "\tDetails:  %log%"
 172
 173     def check(self):
 174         if len(p4CmdList("opened ...")) > 0:
 175             die("You have files opened with perforce! Close them before starting the sync.")
 176
 177     def start(self):
 178         if len(self.config) > 0 and not self.reset:
 179             die("Cannot start sync. Previous sync config found at %s" % self.configFile)
 180
 181         commits = []
 182         for line in os.popen("git-rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines():
 183             commits.append(line[:-1])
 184         commits.reverse()
 185
 186         self.config["commits"] = commits
 187
 188         if not self.applyAsPatch:
 189             print "Creating temporary p4-sync branch from %s ..." % self.origin
 190             system("git checkout -f -b p4-sync %s" % self.origin)
 191
 192     def prepareLogMessage(self, template, message):
 193         result = ""
 194
 195         for line in template.split("\n"):
 196             if line.startswith("#"):
 197                 result += line + "\n"
 198                 continue
 199
 200             substituted = False
 201             for key in self.logSubstitutions.keys():
 202                 if line.find(key) != -1:
 203                     value = self.logSubstitutions[key]
 204                     value = value.replace("%log%", message)
 205                     if value != "@remove@":
 206                         result += line.replace(key, value) + "\n"
 207                     substituted = True
 208                     break
 209
 210             if not substituted:
 211                 result += line + "\n"
 212
 213         return result
 214
 215     def apply(self, id):
 216         print "Applying %s" % (os.popen("git-log --max-count=1 --pretty=oneline %s" % id).read())
 217         diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines()
 218         filesToAdd = set()
 219         filesToDelete = set()
 220         for line in diff:
 221             modifier = line[0]
 222             path = line[1:].strip()
 223             if modifier == "M":
 224                 system("p4 edit %s" % path)
 225             elif modifier == "A":
 226                 filesToAdd.add(path)
 227                 if path in filesToDelete:
 228                     filesToDelete.remove(path)
 229             elif modifier == "D":
 230                 filesToDelete.add(path)
 231                 if path in filesToAdd:
 232                     filesToAdd.remove(path)
 233             else:
 234                 die("unknown modifier %s for %s" % (modifier, path))
 235
 236         if self.applyAsPatch:
 237             system("git-diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\" | patch -p1" % (id, id))
 238         else:
 239             system("git-diff-files --name-only -z | git-update-index --remove -z --stdin")
 240             system("git cherry-pick --no-commit \"%s\"" % id)
 241
 242         for f in filesToAdd:
 243             system("p4 add %s" % f)
 244         for f in filesToDelete:
 245             system("p4 revert %s" % f)
 246             system("p4 delete %s" % f)
 247
 248         logMessage = extractLogMessageFromGitCommit(id)
 249         logMessage = logMessage.replace("\n", "\n\t")
 250         logMessage = logMessage[:-1]
 251
 252         template = os.popen("p4 change -o").read()
 253
 254         if self.interactive:
 255             submitTemplate = self.prepareLogMessage(template, logMessage)
 256             diff = os.popen("p4 diff -du ...").read()
 257
 258             for newFile in filesToAdd:
 259                 diff += "==== new file ====\n"
 260                 diff += "--- /dev/null\n"
 261                 diff += "+++ %s\n" % newFile
 262                 f = open(newFile, "r")
 263                 for line in f.readlines():
 264                     diff += "+" + line
 265                 f.close()
 266
 267             separatorLine = "######## everything below this line is just the diff #######\n"
 268
 269             response = "e"
 270             firstIteration = True
 271             while response == "e":
 272                 if not firstIteration:
 273                     response = raw_input("Do you want to submit this change (y/e/n)? ")
 274                 firstIteration = False
 275                 if response == "e":
 276                     [handle, fileName] = tempfile.mkstemp()
 277                     tmpFile = os.fdopen(handle, "w+")
 278                     tmpFile.write(submitTemplate + separatorLine + diff)
 279                     tmpFile.close()
 280                     editor = os.environ.get("EDITOR", "vi")
 281                     system(editor + " " + fileName)
 282                     tmpFile = open(fileName, "r")
 283                     message = tmpFile.read()
 284                     tmpFile.close()
 285                     os.remove(fileName)
 286                     submitTemplate = message[:message.index(separatorLine)]
 287
 288             if response == "y" or response == "yes":
 289                if self.dryRun:
 290                    print submitTemplate
 291                    raw_input("Press return to continue...")
 292                else:
 293                     pipe = os.popen("p4 submit -i", "w")
 294                     pipe.write(submitTemplate)
 295                     pipe.close()
 296             else:
 297                 print "Not submitting!"
 298                 self.interactive = False
 299         else:
 300             fileName = "submit.txt"
 301             file = open(fileName, "w+")
 302             file.write(self.prepareLogMessage(template, logMessage))
 303             file.close()
 304             print "Perforce submit template written as %s. Please review/edit and then use p4 submit -i < %s to submit directly!" % (fileName, fileName)
 305
 306     def run(self, args):
 307         global gitdir
 308         # make gitdir absolute so we can cd out into the perforce checkout
 309         gitdir = os.path.abspath(gitdir)
 310         os.environ["GIT_DIR"] = gitdir
 311         depotPath = ""
 312         if gitBranchExists("p4"):
 313             [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("p4"))
 314         if len(depotPath) == 0 and gitBranchExists("origin"):
 315             [depotPath, dummy] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit("origin"))
 316
 317         if len(depotPath) == 0:
 318             print "Internal error: cannot locate perforce depot path from existing branches"
 319             sys.exit(128)
 320
 321         if not depotPath.endswith("/"):
 322             depotPath += "/"
 323         clientPath = p4Cmd("where %s..." % depotPath).get("path")
 324         if clientPath.endswith("..."):
 325             clientPath = clientPath[:-3]
 326
 327         if len(clientPath) == 0:
 328             print "Error: Cannot locate perforce checkout of %s in client view" % depotPath
 329             sys.exit(128)
 330
 331         print "Perforce checkout for depot path %s located at %s" % (depotPath, clientPath)
 332         os.chdir(clientPath)
 333         response = raw_input("Do you want to sync %s with p4 sync? (y/n)" % clientPath)
 334         if response == "y" or response == "yes":
 335             system("p4 sync ...")
 336
 337         if len(self.origin) == 0:
 338             if gitBranchExists("p4"):
 339                 self.origin = "p4"
 340             else:
 341                 self.origin = "origin"
 342
 343         if self.reset:
 344             self.firstTime = True
 345
 346         if len(self.substFile) > 0:
 347             for line in open(self.substFile, "r").readlines():
 348                 tokens = line[:-1].split("=")
 349                 self.logSubstitutions[tokens[0]] = tokens[1]
 350
 351         if len(self.master) == 0:
 352             self.master = currentGitBranch()
 353             if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)):
 354                 die("Detecting current git branch failed!")
 355
 356         self.check()
 357         self.configFile = gitdir + "/p4-git-sync.cfg"
 358         self.config = shelve.open(self.configFile, writeback=True)
 359
 360         if self.firstTime:
 361             self.start()
 362
 363         commits = self.config.get("commits", [])
 364
 365         while len(commits) > 0:
 366             self.firstTime = False
 367             commit = commits[0]
 368             commits = commits[1:]
 369             self.config["commits"] = commits
 370             self.apply(commit)
 371             if not self.interactive:
 372                 break
 373
 374         self.config.close()
 375
 376         if len(commits) == 0:
 377             if self.firstTime:
 378                 print "No changes found to apply between %s and current HEAD" % self.origin
 379             else:
 380                 print "All changes applied!"
 381                 if not self.applyAsPatch:
 382                     print "Deleting temporary p4-sync branch and going back to %s" % self.master
 383                     system("git checkout %s" % self.master)
 384                     system("git branch -D p4-sync")
 385                     print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..."
 386                     system("p4 edit ... >/dev/null")
 387                     system("p4 revert ... >/dev/null")
 388             os.remove(self.configFile)
 389
 390         return True
 391
 392 class GitSync(Command):
 393     def __init__(self):
 394         Command.__init__(self)
 395         self.options = [
 396                 optparse.make_option("--branch", dest="branch"),
 397                 optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"),
 398                 optparse.make_option("--changesfile", dest="changesFile"),
 399                 optparse.make_option("--silent", dest="silent", action="store_true"),
 400                 optparse.make_option("--known-branches", dest="knownBranches"),
 401                 optparse.make_option("--cache", dest="doCache", action="store_true"),
 402                 optparse.make_option("--command-cache", dest="commandCache", action="store_true")
 403         ]
 404         self.description = """Imports from Perforce into a git repository.\n
 405     example:
 406     //depot/my/project/ -- to import the current head
 407     //depot/my/project/@all -- to import everything
 408     //depot/my/project/@1,6 -- to import only from revision 1 to 6
 409
 410     (a ... is not needed in the path p4 specification, it's added implicitly)"""
 411
 412         self.usage += " //depot/path[@revRange]"
 413
 414         self.dataCache = False
 415         self.commandCache = False
 416         self.silent = False
 417         self.knownBranches = Set()
 418         self.createdBranches = Set()
 419         self.committedChanges = Set()
 420         self.branch = ""
 421         self.detectBranches = False
 422         self.changesFile = ""
 423
 424     def p4File(self, depotPath):
 425         return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read()
 426
 427     def extractFilesFromCommit(self, commit):
 428         files = []
 429         fnum = 0
 430         while commit.has_key("depotFile%s" % fnum):
 431             path =  commit["depotFile%s" % fnum]
 432             if not path.startswith(self.globalPrefix):
 433     #            if not self.silent:
 434     #                print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.globalPrefix, change)
 435                 fnum = fnum + 1
 436                 continue
 437
 438             file = {}
 439             file["path"] = path
 440             file["rev"] = commit["rev%s" % fnum]
 441             file["action"] = commit["action%s" % fnum]
 442             file["type"] = commit["type%s" % fnum]
 443             files.append(file)
 444             fnum = fnum + 1
 445         return files
 446
 447     def isSubPathOf(self, first, second):
 448         if not first.startswith(second):
 449             return False
 450         if first == second:
 451             return True
 452         return first[len(second)] == "/"
 453
 454     def branchesForCommit(self, files):
 455         branches = Set()
 456
 457         for file in files:
 458             relativePath = file["path"][len(self.globalPrefix):]
 459             # strip off the filename
 460             relativePath = relativePath[0:relativePath.rfind("/")]
 461
 462     #        if len(branches) == 0:
 463     #            branches.add(relativePath)
 464     #            knownBranches.add(relativePath)
 465     #            continue
 466
 467             ###### this needs more testing :)
 468             knownBranch = False
 469             for branch in branches:
 470                 if relativePath == branch:
 471                     knownBranch = True
 472                     break
 473     #            if relativePath.startswith(branch):
 474                 if self.isSubPathOf(relativePath, branch):
 475                     knownBranch = True
 476                     break
 477     #            if branch.startswith(relativePath):
 478                 if self.isSubPathOf(branch, relativePath):
 479                     branches.remove(branch)
 480                     break
 481
 482             if knownBranch:
 483                 continue
 484
 485             for branch in knownBranches:
 486                 #if relativePath.startswith(branch):
 487                 if self.isSubPathOf(relativePath, branch):
 488                     if len(branches) == 0:
 489                         relativePath = branch
 490                     else:
 491                         knownBranch = True
 492                     break
 493
 494             if knownBranch:
 495                 continue
 496
 497             branches.add(relativePath)
 498             self.knownBranches.add(relativePath)
 499
 500         return branches
 501
 502     def findBranchParent(self, branchPrefix, files):
 503         for file in files:
 504             path = file["path"]
 505             if not path.startswith(branchPrefix):
 506                 continue
 507             action = file["action"]
 508             if action != "integrate" and action != "branch":
 509                 continue
 510             rev = file["rev"]
 511             depotPath = path + "#" + rev
 512
 513             log = p4CmdList("filelog \"%s\"" % depotPath)
 514             if len(log) != 1:
 515                 print "eek! I got confused by the filelog of %s" % depotPath
 516                 sys.exit(1);
 517
 518             log = log[0]
 519             if log["action0"] != action:
 520                 print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 521                 sys.exit(1);
 522
 523             branchAction = log["how0,0"]
 524     #        if branchAction == "branch into" or branchAction == "ignored":
 525     #            continue # ignore for branching
 526
 527             if not branchAction.endswith(" from"):
 528                 continue # ignore for branching
 529     #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 530     #            sys.exit(1);
 531
 532             source = log["file0,0"]
 533             if source.startswith(branchPrefix):
 534                 continue
 535
 536             lastSourceRev = log["erev0,0"]
 537
 538             sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 539             if len(sourceLog) != 1:
 540                 print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 541                 sys.exit(1);
 542             sourceLog = sourceLog[0]
 543
 544             relPath = source[len(self.globalPrefix):]
 545             # strip off the filename
 546             relPath = relPath[0:relPath.rfind("/")]
 547
 548             for branch in self.knownBranches:
 549                 if self.isSubPathOf(relPath, branch):
 550     #                print "determined parent branch branch %s due to change in file %s" % (branch, source)
 551                     return branch
 552     #            else:
 553     #                print "%s is not a subpath of branch %s" % (relPath, branch)
 554
 555         return ""
 556
 557     def commit(self, details, files, branch, branchPrefix, parent = "", merged = ""):
 558         epoch = details["time"]
 559         author = details["user"]
 560
 561         self.gitStream.write("commit %s\n" % branch)
 562     #    gitStream.write("mark :%s\n" % details["change"])
 563         self.committedChanges.add(int(details["change"]))
 564         committer = ""
 565         if author in self.users:
 566             committer = "%s %s %s" % (self.users[author], epoch, self.tz)
 567         else:
 568             committer = "%s <a@b> %s %s" % (author, epoch, self.tz)
 569
 570         self.gitStream.write("committer %s\n" % committer)
 571
 572         self.gitStream.write("data <<EOT\n")
 573         self.gitStream.write(details["desc"])
 574         self.gitStream.write("\n[git-p4: depot-path = \"%s\": change = %s]\n" % (branchPrefix, details["change"]))
 575         self.gitStream.write("EOT\n\n")
 576
 577         if len(parent) > 0:
 578             self.gitStream.write("from %s\n" % parent)
 579
 580         if len(merged) > 0:
 581             self.gitStream.write("merge %s\n" % merged)
 582
 583         for file in files:
 584             path = file["path"]
 585             if not path.startswith(branchPrefix):
 586     #            if not silent:
 587     #                print "\nchanged files: ignoring path %s outside of branch prefix %s in change %s" % (path, branchPrefix, details["change"])
 588                 continue
 589             rev = file["rev"]
 590             depotPath = path + "#" + rev
 591             relPath = path[len(branchPrefix):]
 592             action = file["action"]
 593
 594             if file["type"] == "apple":
 595                 print "\nfile %s is a strange apple file that forks. Ignoring!" % path
 596                 continue
 597
 598             if action == "delete":
 599                 self.gitStream.write("D %s\n" % relPath)
 600             else:
 601                 mode = 644
 602                 if file["type"].startswith("x"):
 603                     mode = 755
 604
 605                 data = self.p4File(depotPath)
 606
 607                 self.gitStream.write("M %s inline %s\n" % (mode, relPath))
 608                 self.gitStream.write("data %s\n" % len(data))
 609                 self.gitStream.write(data)
 610                 self.gitStream.write("\n")
 611
 612         self.gitStream.write("\n")
 613
 614         self.lastChange = int(details["change"])
 615
 616     def extractFilesInCommitToBranch(self, files, branchPrefix):
 617         newFiles = []
 618
 619         for file in files:
 620             path = file["path"]
 621             if path.startswith(branchPrefix):
 622                 newFiles.append(file)
 623
 624         return newFiles
 625
 626     def findBranchSourceHeuristic(self, files, branch, branchPrefix):
 627         for file in files:
 628             action = file["action"]
 629             if action != "integrate" and action != "branch":
 630                 continue
 631             path = file["path"]
 632             rev = file["rev"]
 633             depotPath = path + "#" + rev
 634
 635             log = p4CmdList("filelog \"%s\"" % depotPath)
 636             if len(log) != 1:
 637                 print "eek! I got confused by the filelog of %s" % depotPath
 638                 sys.exit(1);
 639
 640             log = log[0]
 641             if log["action0"] != action:
 642                 print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action)
 643                 sys.exit(1);
 644
 645             branchAction = log["how0,0"]
 646
 647             if not branchAction.endswith(" from"):
 648                 continue # ignore for branching
 649     #            print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction)
 650     #            sys.exit(1);
 651
 652             source = log["file0,0"]
 653             if source.startswith(branchPrefix):
 654                 continue
 655
 656             lastSourceRev = log["erev0,0"]
 657
 658             sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev))
 659             if len(sourceLog) != 1:
 660                 print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev)
 661                 sys.exit(1);
 662             sourceLog = sourceLog[0]
 663
 664             relPath = source[len(self.globalPrefix):]
 665             # strip off the filename
 666             relPath = relPath[0:relPath.rfind("/")]
 667
 668             for candidate in self.knownBranches:
 669                 if self.isSubPathOf(relPath, candidate) and candidate != branch:
 670                     return candidate
 671
 672         return ""
 673
 674     def changeIsBranchMerge(self, sourceBranch, destinationBranch, change):
 675         sourceFiles = {}
 676         for file in p4CmdList("files %s...@%s" % (self.globalPrefix + sourceBranch + "/", change)):
 677             if file["action"] == "delete":
 678                 continue
 679             sourceFiles[file["depotFile"]] = file
 680
 681         destinationFiles = {}
 682         for file in p4CmdList("files %s...@%s" % (self.globalPrefix + destinationBranch + "/", change)):
 683             destinationFiles[file["depotFile"]] = file
 684
 685         for fileName in sourceFiles.keys():
 686             integrations = []
 687             deleted = False
 688             integrationCount = 0
 689             for integration in p4CmdList("integrated \"%s\"" % fileName):
 690                 toFile = integration["fromFile"] # yes, it's true, it's fromFile
 691                 if not toFile in destinationFiles:
 692                     continue
 693                 destFile = destinationFiles[toFile]
 694                 if destFile["action"] == "delete":
 695     #                print "file %s has been deleted in %s" % (fileName, toFile)
 696                     deleted = True
 697                     break
 698                 integrationCount += 1
 699                 if integration["how"] == "branch from":
 700                     continue
 701
 702                 if int(integration["change"]) == change:
 703                     integrations.append(integration)
 704                     continue
 705                 if int(integration["change"]) > change:
 706                     continue
 707
 708                 destRev = int(destFile["rev"])
 709
 710                 startRev = integration["startFromRev"][1:]
 711                 if startRev == "none":
 712                     startRev = 0
 713                 else:
 714                     startRev = int(startRev)
 715
 716                 endRev = integration["endFromRev"][1:]
 717                 if endRev == "none":
 718                     endRev = 0
 719                 else:
 720                     endRev = int(endRev)
 721
 722                 initialBranch = (destRev == 1 and integration["how"] != "branch into")
 723                 inRange = (destRev >= startRev and destRev <= endRev)
 724                 newer = (destRev > startRev and destRev > endRev)
 725
 726                 if initialBranch or inRange or newer:
 727                     integrations.append(integration)
 728
 729             if deleted:
 730                 continue
 731
 732             if len(integrations) == 0 and integrationCount > 1:
 733                 print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch)
 734                 return False
 735
 736         return True
 737
 738     def getUserMap(self):
 739         self.users = {}
 740
 741         for output in p4CmdList("users"):
 742             if not output.has_key("User"):
 743                 continue
 744             self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">"
 745
 746     def run(self, args):
 747         self.globalPrefix = ""
 748         self.changeRange = ""
 749         self.initialParent = ""
 750         self.tagLastChange = True
 751
 752         if len(self.branch) == 0:
 753             self.branch = "p4"
 754             if len(args) == 0:
 755                 if not gitBranchExists(self.branch) and gitBranchExists("origin"):
 756                     if not self.silent:
 757                         print "Creating %s branch in git repository based on origin" % self.branch
 758                     system("git branch %s origin" % self.branch)
 759
 760                 [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch))
 761                 if len(self.previousDepotPath) > 0 and len(p4Change) > 0:
 762                     p4Change = int(p4Change) + 1
 763                     self.globalPrefix = self.previousDepotPath
 764                     self.changeRange = "@%s,#head" % p4Change
 765                     self.initialParent = self.branch
 766                     self.tagLastChange = False
 767                     if not self.silent:
 768                         print "Performing incremental import into %s git branch" % self.branch
 769
 770         self.branch = "refs/heads/" + self.branch
 771
 772         if len(self.globalPrefix) == 0:
 773             self.globalPrefix = self.previousDepotPath = os.popen("git-repo-config --get p4.depotpath").read()
 774
 775         if len(self.globalPrefix) != 0:
 776             self.globalPrefix = self.globalPrefix[:-1]
 777
 778         if len(args) == 0 and len(self.globalPrefix) != 0:
 779             if not self.silent:
 780                 print "Depot path: %s" % self.globalPrefix
 781         elif len(args) != 1:
 782             return False
 783         else:
 784             if len(self.globalPrefix) != 0 and self.globalPrefix != args[0]:
 785                 print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.globalPrefix, args[0])
 786                 sys.exit(1)
 787             self.globalPrefix = args[0]
 788
 789         self.revision = ""
 790         self.users = {}
 791         self.lastChange = 0
 792         self.initialTag = ""
 793
 794         if self.globalPrefix.find("@") != -1:
 795             atIdx = self.globalPrefix.index("@")
 796             self.changeRange = self.globalPrefix[atIdx:]
 797             if self.changeRange == "@all":
 798                 self.changeRange = ""
 799             elif self.changeRange.find(",") == -1:
 800                 self.revision = self.changeRange
 801                 self.changeRange = ""
 802             self.globalPrefix = self.globalPrefix[0:atIdx]
 803         elif self.globalPrefix.find("#") != -1:
 804             hashIdx = self.globalPrefix.index("#")
 805             self.revision = self.globalPrefix[hashIdx:]
 806             self.globalPrefix = self.globalPrefix[0:hashIdx]
 807         elif len(self.previousDepotPath) == 0:
 808             self.revision = "#head"
 809
 810         if self.globalPrefix.endswith("..."):
 811             self.globalPrefix = self.globalPrefix[:-3]
 812
 813         if not self.globalPrefix.endswith("/"):
 814             self.globalPrefix += "/"
 815
 816         self.getUserMap()
 817
 818         if len(self.changeRange) == 0:
 819             try:
 820                 sout, sin, serr = popen2.popen3("git-name-rev --tags `git-rev-parse %s`" % self.branch)
 821                 output = sout.read()
 822                 if output.endswith("\n"):
 823                     output = output[:-1]
 824                 tagIdx = output.index(" tags/p4/")
 825                 caretIdx = output.find("^")
 826                 endPos = len(output)
 827                 if caretIdx != -1:
 828                     endPos = caretIdx
 829                 self.rev = int(output[tagIdx + 9 : endPos]) + 1
 830                 self.changeRange = "@%s,#head" % self.rev
 831                 self.initialParent = os.popen("git-rev-parse %s" % self.branch).read()[:-1]
 832                 self.initialTag = "p4/%s" % (int(self.rev) - 1)
 833             except:
 834                 pass
 835
 836         self.tz = - time.timezone / 36
 837         tzsign = ("%s" % self.tz)[0]
 838         if tzsign != '+' and tzsign != '-':
 839             self.tz = "+" + ("%s" % self.tz)
 840
 841         self.gitOutput, self.gitStream, self.gitError = popen2.popen3("git-fast-import")
 842
 843         if len(self.revision) > 0:
 844             print "Doing initial import of %s from revision %s" % (self.globalPrefix, self.revision)
 845
 846             details = { "user" : "git perforce import user", "time" : int(time.time()) }
 847             details["desc"] = "Initial import of %s from the state at revision %s" % (self.globalPrefix, self.revision)
 848             details["change"] = self.revision
 849             newestRevision = 0
 850
 851             fileCnt = 0
 852             for info in p4CmdList("files %s...%s" % (self.globalPrefix, self.revision)):
 853                 change = int(info["change"])
 854                 if change > newestRevision:
 855                     newestRevision = change
 856
 857                 if info["action"] == "delete":
 858                     fileCnt = fileCnt + 1
 859                     continue
 860
 861                 for prop in [ "depotFile", "rev", "action", "type" ]:
 862                     details["%s%s" % (prop, fileCnt)] = info[prop]
 863
 864                 fileCnt = fileCnt + 1
 865
 866             details["change"] = newestRevision
 867
 868             try:
 869                 self.commit(details, self.extractFilesFromCommit(details), self.branch, self.globalPrefix)
 870             except IOError:
 871                 print self.gitError.read()
 872
 873         else:
 874             changes = []
 875
 876             if len(self.changesFile) > 0:
 877                 output = open(self.changesFile).readlines()
 878                 changeSet = Set()
 879                 for line in output:
 880                     changeSet.add(int(line))
 881
 882                 for change in changeSet:
 883                     changes.append(change)
 884
 885                 changes.sort()
 886             else:
 887                 output = os.popen("p4 changes %s...%s" % (self.globalPrefix, self.changeRange)).readlines()
 888
 889                 for line in output:
 890                     changeNum = line.split(" ")[1]
 891                     changes.append(changeNum)
 892
 893                 changes.reverse()
 894
 895             if len(changes) == 0:
 896                 if not self.silent:
 897                     print "no changes to import!"
 898                 sys.exit(1)
 899
 900             cnt = 1
 901             for change in changes:
 902                 description = p4Cmd("describe %s" % change)
 903
 904                 if not self.silent:
 905                     sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes)))
 906                     sys.stdout.flush()
 907                 cnt = cnt + 1
 908
 909                 try:
 910                     files = self.extractFilesFromCommit(description)
 911                     if self.detectBranches:
 912                         for branch in self.branchesForCommit(files):
 913                             self.knownBranches.add(branch)
 914                             branchPrefix = self.globalPrefix + branch + "/"
 915
 916                             filesForCommit = self.extractFilesInCommitToBranch(files, branchPrefix)
 917
 918                             merged = ""
 919                             parent = ""
 920                             ########### remove cnt!!!
 921                             if branch not in self.createdBranches and cnt > 2:
 922                                 self.createdBranches.add(branch)
 923                                 parent = self.findBranchParent(branchPrefix, files)
 924                                 if parent == branch:
 925                                     parent = ""
 926             #                    elif len(parent) > 0:
 927             #                        print "%s branched off of %s" % (branch, parent)
 928
 929                             if len(parent) == 0:
 930                                 merged = self.findBranchSourceHeuristic(filesForCommit, branch, branchPrefix)
 931                                 if len(merged) > 0:
 932                                     print "change %s could be a merge from %s into %s" % (description["change"], merged, branch)
 933                                     if not self.changeIsBranchMerge(merged, branch, int(description["change"])):
 934                                         merged = ""
 935
 936                             branch = "refs/heads/" + branch
 937                             if len(parent) > 0:
 938                                 parent = "refs/heads/" + parent
 939                             if len(merged) > 0:
 940                                 merged = "refs/heads/" + merged
 941                             self.commit(description, files, branch, branchPrefix, parent, merged)
 942                     else:
 943                         self.commit(description, files, self.branch, self.globalPrefix, self.initialParent)
 944                         self.initialParent = ""
 945                 except IOError:
 946                     print self.gitError.read()
 947                     sys.exit(1)
 948
 949         if not self.silent:
 950             print ""
 951
 952         if self.tagLastChange:
 953             self.gitStream.write("reset refs/tags/p4/%s\n" % self.lastChange)
 954             self.gitStream.write("from %s\n\n" % self.branch);
 955
 956
 957         self.gitStream.close()
 958         self.gitOutput.close()
 959         self.gitError.close()
 960
 961         os.popen("git-repo-config p4.depotpath %s" % self.globalPrefix).read()
 962         if len(self.initialTag) > 0:
 963             os.popen("git tag -d %s" % self.initialTag).read()
 964
 965         return True
 966
 967 class HelpFormatter(optparse.IndentedHelpFormatter):
 968     def __init__(self):
 969         optparse.IndentedHelpFormatter.__init__(self)
 970
 971     def format_description(self, description):
 972         if description:
 973             return description + "\n"
 974         else:
 975             return ""
 976
 977 def printUsage(commands):
 978     print "usage: %s <command> [options]" % sys.argv[0]
 979     print ""
 980     print "valid commands: %s" % ", ".join(commands)
 981     print ""
 982     print "Try %s <command> --help for command specific help." % sys.argv[0]
 983     print ""
 984
 985 commands = {
 986     "debug" : P4Debug(),
 987     "clean-tags" : P4CleanTags(),
 988     "submit" : P4Sync(),
 989     "sync" : GitSync()
 990 }
 991
 992 if len(sys.argv[1:]) == 0:
 993     printUsage(commands.keys())
 994     sys.exit(2)
 995
 996 cmd = ""
 997 cmdName = sys.argv[1]
 998 try:
 999     cmd = commands[cmdName]
1000 except KeyError:
1001     print "unknown command %s" % cmdName
1002     print ""
1003     printUsage(commands.keys())
1004     sys.exit(2)
1005
1006 options = cmd.options
1007 cmd.gitdir = gitdir
1008 options.append(optparse.make_option("--git-dir", dest="gitdir"))
1009
1010 parser = optparse.OptionParser(cmd.usage.replace("%prog", "%prog " + cmdName),
1011                                options,
1012                                description = cmd.description,
1013                                formatter = HelpFormatter())
1014
1015 (cmd, args) = parser.parse_args(sys.argv[2:], cmd);
1016
1017 gitdir = cmd.gitdir
1018 if len(gitdir) == 0:
1019     gitdir = ".git"
1020     if not isValidGitDir(gitdir):
1021         cdup = os.popen("git-rev-parse --show-cdup").read()[:-1]
1022         if isValidGitDir(cdup + "/" + gitdir):
1023             os.chdir(cdup)
1024
1025 if not isValidGitDir(gitdir):
1026     if isValidGitDir(gitdir + "/.git"):
1027         gitdir += "/.git"
1028     else:
1029         die("fatal: cannot locate git repository at %s" % gitdir)
1030
1031 os.environ["GIT_DIR"] = gitdir
1032
1033 if not cmd.run(args):
1034     parser.print_help()
1035