X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=contrib%2Ffast-import%2Fgit-p4;h=e8a5c1fa316f2ac5be4fa32b145d233db31252a5;hb=4280e5333354c6dddcd994bdacd3c6a11ac2da5e;hp=72fa48af04a7dc4f3c736ad4c0642631dab41370;hpb=10c3211b81a2f67c3853900e462333933f910696;p=git.git diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 72fa48af0..e8a5c1fa3 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -7,13 +7,19 @@ # 2007 Trolltech ASA # License: MIT # +# TODO: * Consider making --with-origin the default, assuming that the git +# protocol is always more efficient. (needs manual testing first :) +# -import optparse, sys, os, marshal, popen2, shelve -import tempfile, getopt, sha, os.path, time +import optparse, sys, os, marshal, popen2, subprocess, shelve +import tempfile, getopt, sha, os.path, time, platform from sets import Set; gitdir = os.environ.get("GIT_DIR", "") +def mypopen(command): + return os.popen(command, "rb"); + def p4CmdList(cmd): cmd = "p4 -G %s" % cmd pipe = os.popen(cmd, "rb") @@ -25,7 +31,11 @@ def p4CmdList(cmd): result.append(entry) except EOFError: pass - pipe.close() + exitCode = pipe.close() + if exitCode != None: + entry = {} + entry["p4ExitCode"] = exitCode + result.append(entry) return result @@ -40,6 +50,8 @@ def p4Where(depotPath): if not depotPath.endswith("/"): depotPath += "/" output = p4Cmd("where %s..." % depotPath) + if output["code"] == "error": + return "" clientPath = "" if "path" in output: clientPath = output.get("path") @@ -57,13 +69,16 @@ def die(msg): sys.exit(1) def currentGitBranch(): - return os.popen("git name-rev HEAD").read().split(" ")[1][:-1] + return mypopen("git name-rev HEAD").read().split(" ")[1][:-1] def isValidGitDir(path): if os.path.exists(path + "/HEAD") and os.path.exists(path + "/refs") and os.path.exists(path + "/objects"): return True; return False +def parseRevision(ref): + return mypopen("git rev-parse %s" % ref).read()[:-1] + def system(cmd): if os.system(cmd) != 0: die("command failed: %s" % cmd) @@ -71,10 +86,10 @@ def system(cmd): def extractLogMessageFromGitCommit(commit): logMessage = "" foundTitle = False - for log in os.popen("git cat-file commit %s" % commit).readlines(): + for log in mypopen("git cat-file commit %s" % commit).readlines(): if not foundTitle: if len(log) == 1: - foundTitle = 1 + foundTitle = True continue logMessage += log @@ -100,9 +115,8 @@ def extractDepotPathAndChangeFromGitLog(log): return values.get("depot-path"), values.get("change") def gitBranchExists(branch): - if os.system("git rev-parse %s 2>/dev/null >/dev/null" % branch) == 0: - return True - return False + proc = subprocess.Popen(["git", "rev-parse", branch], stderr=subprocess.PIPE, stdout=subprocess.PIPE); + return proc.wait() == 0; class Command: def __init__(self): @@ -122,42 +136,55 @@ class P4Debug(Command): print output return True -class P4CleanTags(Command): +class P4RollBack(Command): def __init__(self): Command.__init__(self) self.options = [ -# optparse.make_option("--branch", dest="branch", default="refs/heads/master") + optparse.make_option("--verbose", dest="verbose", action="store_true"), + optparse.make_option("--local", dest="rollbackLocalBranches", action="store_true") ] - self.description = "A tool to remove stale unused tags from incremental perforce imports." - def run(self, args): - branch = currentGitBranch() - print "Cleaning out stale p4 import tags..." - sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % branch) - output = sout.read() - try: - tagIdx = output.index(" tags/p4/") - except: - print "Cannot find any p4/* tag. Nothing to do." - sys.exit(0) + self.description = "A tool to debug the multi-branch import. Don't use :)" + self.verbose = False + self.rollbackLocalBranches = False - try: - caretIdx = output.index("^") - except: - caretIdx = len(output) - 1 - rev = int(output[tagIdx + 9 : caretIdx]) + def run(self, args): + if len(args) != 1: + return False + maxChange = int(args[0]) - allTags = os.popen("git tag -l p4/").readlines() - for i in range(len(allTags)): - allTags[i] = int(allTags[i][3:-1]) + if "p4ExitCode" in p4Cmd("changes -m 1"): + die("Problems executing p4"); - allTags.sort() + if self.rollbackLocalBranches: + refPrefix = "refs/heads/" + lines = mypopen("git rev-parse --symbolic --branches").readlines() + else: + refPrefix = "refs/remotes/" + lines = mypopen("git rev-parse --symbolic --remotes").readlines() + + for line in lines: + if self.rollbackLocalBranches or (line.startswith("p4/") and line != "p4/HEAD\n"): + ref = refPrefix + line[:-1] + log = extractLogMessageFromGitCommit(ref) + depotPath, change = extractDepotPathAndChangeFromGitLog(log) + changed = False + + if len(p4Cmd("changes -m 1 %s...@%s" % (depotPath, maxChange))) == 0: + print "Branch %s did not exist at change %s, deleting." % (ref, maxChange) + system("git update-ref -d %s `git rev-parse %s`" % (ref, ref)) + continue - allTags.remove(rev) + while len(change) > 0 and int(change) > maxChange: + changed = True + if self.verbose: + print "%s is at %s ; rewinding towards %s" % (ref, change, maxChange) + system("git update-ref %s \"%s^\"" % (ref, ref)) + log = extractLogMessageFromGitCommit(ref) + depotPath, change = extractDepotPathAndChangeFromGitLog(log) - for rev in allTags: - print os.popen("git tag -d p4/%s" % rev).read() + if changed: + print "%s rewound to %s" % (ref, change) - print "%s tags removed." % len(allTags) return True class P4Submit(Command): @@ -170,7 +197,7 @@ class P4Submit(Command): optparse.make_option("--log-substitutions", dest="substFile"), optparse.make_option("--noninteractive", action="store_false"), optparse.make_option("--dry-run", action="store_true"), - optparse.make_option("--apply-as-patch", action="store_true", dest="applyAsPatch") + optparse.make_option("--direct", dest="directSubmit", action="store_true"), ] self.description = "Submit changes from git to the perforce depot." self.usage += " [name of git branch to submit into perforce depot]" @@ -181,7 +208,7 @@ class P4Submit(Command): self.substFile = "" self.firstTime = True self.origin = "" - self.applyAsPatch = True + self.directSubmit = False self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" @@ -193,19 +220,18 @@ class P4Submit(Command): def start(self): if len(self.config) > 0 and not self.reset: - die("Cannot start sync. Previous sync config found at %s" % self.configFile) + die("Cannot start sync. Previous sync config found at %s\nIf you want to start submitting again from scratch maybe you want to call git-p4 submit --reset" % self.configFile) commits = [] - for line in os.popen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): - commits.append(line[:-1]) - commits.reverse() + if self.directSubmit: + commits.append("0") + else: + for line in mypopen("git rev-list --no-merges %s..%s" % (self.origin, self.master)).readlines(): + commits.append(line[:-1]) + commits.reverse() self.config["commits"] = commits - if not self.applyAsPatch: - print "Creating temporary p4-sync branch from %s ..." % self.origin - system("git checkout -f -b p4-sync %s" % self.origin) - def prepareLogMessage(self, template, message): result = "" @@ -230,15 +256,21 @@ class P4Submit(Command): return result def apply(self, id): - print "Applying %s" % (os.popen("git log --max-count=1 --pretty=oneline %s" % id).read()) - diff = os.popen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() + if self.directSubmit: + print "Applying local change in working directory/index" + diff = self.diffStatus + else: + print "Applying %s" % (mypopen("git log --max-count=1 --pretty=oneline %s" % id).read()) + diff = mypopen("git diff-tree -r --name-status \"%s^\" \"%s\"" % (id, id)).readlines() filesToAdd = set() filesToDelete = set() + editedFiles = set() for line in diff: modifier = line[0] path = line[1:].strip() if modifier == "M": - system("p4 edit %s" % path) + system("p4 edit \"%s\"" % path) + editedFiles.add(path) elif modifier == "A": filesToAdd.add(path) if path in filesToDelete: @@ -250,11 +282,38 @@ class P4Submit(Command): else: die("unknown modifier %s for %s" % (modifier, path)) - if self.applyAsPatch: - system("git diff-tree -p --diff-filter=ACMRTUXB \"%s^\" \"%s\" | patch -p1" % (id, id)) + if self.directSubmit: + diffcmd = "cat \"%s\"" % self.diffFile else: - system("git diff-files --name-only -z | git update-index --remove -z --stdin") - system("git cherry-pick --no-commit \"%s\"" % id) + diffcmd = "git format-patch -k --stdout \"%s^\"..\"%s\"" % (id, id) + patchcmd = diffcmd + " | git apply " + tryPatchCmd = patchcmd + "--check -" + applyPatchCmd = patchcmd + "--check --apply -" + + if os.system(tryPatchCmd) != 0: + print "Unfortunately applying the change failed!" + print "What do you want to do?" + response = "x" + while response != "s" and response != "a" and response != "w": + response = raw_input("[s]kip this patch / [a]pply the patch forcibly and with .rej files / [w]rite the patch to a file (patch.txt) ") + if response == "s": + print "Skipping! Good luck with the next patches..." + return + elif response == "a": + os.system(applyPatchCmd) + if len(filesToAdd) > 0: + print "You may also want to call p4 add on the following files:" + print " ".join(filesToAdd) + if len(filesToDelete): + print "The following files should be scheduled for deletion with p4 delete:" + print " ".join(filesToDelete) + die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue") + elif response == "w": + system(diffcmd + " > patch.txt") + print "Patch saved to patch.txt in %s !" % self.clientPath + die("Please resolve and submit the conflict manually and continue afterwards with git-p4 submit --continue") + + system(applyPatchCmd) for f in filesToAdd: system("p4 add %s" % f) @@ -262,15 +321,17 @@ class P4Submit(Command): system("p4 revert %s" % f) system("p4 delete %s" % f) - logMessage = extractLogMessageFromGitCommit(id) - logMessage = logMessage.replace("\n", "\n\t") - logMessage = logMessage[:-1] + logMessage = "" + if not self.directSubmit: + logMessage = extractLogMessageFromGitCommit(id) + logMessage = logMessage.replace("\n", "\n\t") + logMessage = logMessage[:-1] - template = os.popen("p4 change -o").read() + template = mypopen("p4 change -o").read() if self.interactive: submitTemplate = self.prepareLogMessage(template, logMessage) - diff = os.popen("p4 diff -du ...").read() + diff = mypopen("p4 diff -du ...").read() for newFile in filesToAdd: diff += "==== new file ====\n" @@ -281,22 +342,28 @@ class P4Submit(Command): diff += "+" + line f.close() - separatorLine = "######## everything below this line is just the diff #######\n" + separatorLine = "######## everything below this line is just the diff #######" + if platform.system() == "Windows": + separatorLine += "\r" + separatorLine += "\n" response = "e" firstIteration = True while response == "e": if not firstIteration: - response = raw_input("Do you want to submit this change (y/e/n)? ") + response = raw_input("Do you want to submit this change? [y]es/[e]dit/[n]o/[s]kip ") firstIteration = False if response == "e": [handle, fileName] = tempfile.mkstemp() tmpFile = os.fdopen(handle, "w+") tmpFile.write(submitTemplate + separatorLine + diff) tmpFile.close() - editor = os.environ.get("EDITOR", "vi") + defaultEditor = "vi" + if platform.system() == "Windows": + defaultEditor = "notepad" + editor = os.environ.get("EDITOR", defaultEditor); system(editor + " " + fileName) - tmpFile = open(fileName, "r") + tmpFile = open(fileName, "rb") message = tmpFile.read() tmpFile.close() os.remove(fileName) @@ -307,9 +374,26 @@ class P4Submit(Command): print submitTemplate raw_input("Press return to continue...") else: - pipe = os.popen("p4 submit -i", "w") - pipe.write(submitTemplate) - pipe.close() + if self.directSubmit: + print "Submitting to git first" + os.chdir(self.oldWorkingDirectory) + pipe = os.popen("git commit -a -F -", "wb") + pipe.write(submitTemplate) + pipe.close() + os.chdir(self.clientPath) + + pipe = os.popen("p4 submit -i", "wb") + pipe.write(submitTemplate) + pipe.close() + elif response == "s": + for f in editedFiles: + system("p4 revert \"%s\"" % f); + for f in filesToAdd: + system("p4 revert \"%s\"" % f); + system("rm %s" %f) + for f in filesToDelete: + system("p4 delete \"%s\"" % f); + return else: print "Not submitting!" self.interactive = False @@ -328,7 +412,7 @@ class P4Submit(Command): if len(args) == 0: self.master = currentGitBranch() - if len(self.master) == 0 or not os.path.exists("%s/refs/heads/%s" % (gitdir, self.master)): + if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master): die("Detecting current git branch failed!") elif len(args) == 1: self.master = args[0] @@ -345,15 +429,28 @@ class P4Submit(Command): print "Internal error: cannot locate perforce depot path from existing branches" sys.exit(128) - clientPath = p4Where(depotPath) + self.clientPath = p4Where(depotPath) - if len(clientPath) == 0: + if len(self.clientPath) == 0: print "Error: Cannot locate perforce checkout of %s in client view" % depotPath sys.exit(128) - print "Perforce checkout for depot path %s located at %s" % (depotPath, clientPath) - os.chdir(clientPath) - response = raw_input("Do you want to sync %s with p4 sync? (y/n) " % clientPath) + print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath) + self.oldWorkingDirectory = os.getcwd() + + if self.directSubmit: + self.diffStatus = mypopen("git diff -r --name-status HEAD").readlines() + if len(self.diffStatus) == 0: + print "No changes in working directory to submit." + return True + patch = mypopen("git diff -p --binary --diff-filter=ACMRTUXB HEAD").read() + self.diffFile = gitdir + "/p4-git-diff" + f = open(self.diffFile, "wb") + f.write(patch) + f.close(); + + os.chdir(self.clientPath) + response = raw_input("Do you want to sync %s with p4 sync? [y]es/[n]o " % self.clientPath) if response == "y" or response == "yes": system("p4 sync ...") @@ -391,18 +488,19 @@ class P4Submit(Command): self.config.close() + if self.directSubmit: + os.remove(self.diffFile) + if len(commits) == 0: if self.firstTime: print "No changes found to apply between %s and current HEAD" % self.origin else: print "All changes applied!" - if not self.applyAsPatch: - print "Deleting temporary p4-sync branch and going back to %s" % self.master - system("git checkout %s" % self.master) - system("git branch -D p4-sync") - print "Cleaning out your perforce checkout by doing p4 edit ... ; p4 revert ..." - system("p4 edit ... >/dev/null") - system("p4 revert ... >/dev/null") + os.chdir(self.oldWorkingDirectory) + response = raw_input("Do you want to sync from Perforce now using git-p4 rebase? [y]es/[n]o ") + if response == "y" or response == "yes": + rebase = P4Rebase() + rebase.run([]) os.remove(self.configFile) return True @@ -415,10 +513,11 @@ class P4Sync(Command): optparse.make_option("--detect-branches", dest="detectBranches", action="store_true"), optparse.make_option("--changesfile", dest="changesFile"), optparse.make_option("--silent", dest="silent", action="store_true"), - optparse.make_option("--known-branches", dest="knownBranches"), - optparse.make_option("--data-cache", dest="dataCache", action="store_true"), - optparse.make_option("--command-cache", dest="commandCache", action="store_true"), - optparse.make_option("--detect-labels", dest="detectLabels", action="store_true") + optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), + optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true"), + optparse.make_option("--verbose", dest="verbose", action="store_true"), + optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"), + optparse.make_option("--max-changes", dest="maxChanges") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -430,17 +529,18 @@ class P4Sync(Command): self.usage += " //depot/path[@revRange]" - self.dataCache = False - self.commandCache = False self.silent = False - self.knownBranches = Set() self.createdBranches = Set() self.committedChanges = Set() self.branch = "" self.detectBranches = False self.detectLabels = False self.changesFile = "" - self.tagLastChange = True + self.syncWithOrigin = False + self.verbose = False + self.importIntoRemotes = True + self.maxChanges = "" + self.isWindows = (platform.system() == "Windows") def p4File(self, depotPath): return os.popen("p4 print -q \"%s\"" % depotPath, "rb").read() @@ -450,9 +550,9 @@ class P4Sync(Command): fnum = 0 while commit.has_key("depotFile%s" % fnum): path = commit["depotFile%s" % fnum] - if not path.startswith(self.globalPrefix): + if not path.startswith(self.depotPath): # if not self.silent: - # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.globalPrefix, change) + # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change) fnum = fnum + 1 continue @@ -465,124 +565,48 @@ class P4Sync(Command): fnum = fnum + 1 return files - def isSubPathOf(self, first, second): - if not first.startswith(second): - return False - if first == second: - return True - return first[len(second)] == "/" - - def branchesForCommit(self, files): - branches = Set() + def splitFilesIntoBranches(self, commit): + branches = {} - for file in files: - relativePath = file["path"][len(self.globalPrefix):] - # strip off the filename - relativePath = relativePath[0:relativePath.rfind("/")] - - # if len(branches) == 0: - # branches.add(relativePath) - # knownBranches.add(relativePath) - # continue - - ###### this needs more testing :) - knownBranch = False - for branch in branches: - if relativePath == branch: - knownBranch = True - break - # if relativePath.startswith(branch): - if self.isSubPathOf(relativePath, branch): - knownBranch = True - break - # if branch.startswith(relativePath): - if self.isSubPathOf(branch, relativePath): - branches.remove(branch) - break - - if knownBranch: + fnum = 0 + while commit.has_key("depotFile%s" % fnum): + path = commit["depotFile%s" % fnum] + if not path.startswith(self.depotPath): + # if not self.silent: + # print "\nchanged files: ignoring path %s outside of %s in change %s" % (path, self.depotPath, change) + fnum = fnum + 1 continue - for branch in self.knownBranches: - #if relativePath.startswith(branch): - if self.isSubPathOf(relativePath, branch): - if len(branches) == 0: - relativePath = branch - else: - knownBranch = True - break + file = {} + file["path"] = path + file["rev"] = commit["rev%s" % fnum] + file["action"] = commit["action%s" % fnum] + file["type"] = commit["type%s" % fnum] + fnum = fnum + 1 - if knownBranch: - continue + relPath = path[len(self.depotPath):] - branches.add(relativePath) - self.knownBranches.add(relativePath) + for branch in self.knownBranches.keys(): + if relPath.startswith(branch + "/"): # add a trailing slash so that a commit into qt/4.2foo doesn't end up in qt/4.2 + if branch not in branches: + branches[branch] = [] + branches[branch].append(file) return branches - def findBranchParent(self, branchPrefix, files): - for file in files: - path = file["path"] - if not path.startswith(branchPrefix): - continue - action = file["action"] - if action != "integrate" and action != "branch": - continue - rev = file["rev"] - depotPath = path + "#" + rev - - log = p4CmdList("filelog \"%s\"" % depotPath) - if len(log) != 1: - print "eek! I got confused by the filelog of %s" % depotPath - sys.exit(1); - - log = log[0] - if log["action0"] != action: - print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) - sys.exit(1); - - branchAction = log["how0,0"] - # if branchAction == "branch into" or branchAction == "ignored": - # continue # ignore for branching - - if not branchAction.endswith(" from"): - continue # ignore for branching - # print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) - # sys.exit(1); - - source = log["file0,0"] - if source.startswith(branchPrefix): - continue - - lastSourceRev = log["erev0,0"] - - sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) - if len(sourceLog) != 1: - print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) - sys.exit(1); - sourceLog = sourceLog[0] - - relPath = source[len(self.globalPrefix):] - # strip off the filename - relPath = relPath[0:relPath.rfind("/")] - - for branch in self.knownBranches: - if self.isSubPathOf(relPath, branch): - # print "determined parent branch branch %s due to change in file %s" % (branch, source) - return branch - # else: - # print "%s is not a subpath of branch %s" % (relPath, branch) - - return "" - - def commit(self, details, files, branch, branchPrefix, parent = "", merged = ""): + def commit(self, details, files, branch, branchPrefix, parent = ""): epoch = details["time"] author = details["user"] + if self.verbose: + print "commit into %s" % branch + self.gitStream.write("commit %s\n" % branch) # gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) committer = "" + if author not in self.users: + self.getUserMapFromPerforceServer() if author in self.users: committer = "%s %s %s" % (self.users[author], epoch, self.tz) else: @@ -596,11 +620,10 @@ class P4Sync(Command): self.gitStream.write("EOT\n\n") if len(parent) > 0: + if self.verbose: + print "parent %s" % parent self.gitStream.write("from %s\n" % parent) - if len(merged) > 0: - self.gitStream.write("merge %s\n" % merged) - for file in files: path = file["path"] if not path.startswith(branchPrefix): @@ -625,6 +648,9 @@ class P4Sync(Command): data = self.p4File(depotPath) + if self.isWindows and file["type"].endswith("text"): + data = data.replace("\r\n", "\n") + self.gitStream.write("M %s inline %s\n" % (mode, relPath)) self.gitStream.write("data %s\n" % len(data)) self.gitStream.write(data) @@ -634,12 +660,12 @@ class P4Sync(Command): change = int(details["change"]) - self.lastChange = change - - if change in self.labels: + if self.labels.has_key(change): label = self.labels[change] labelDetails = label[0] labelRevisions = label[1] + if self.verbose: + print "Change %s is labelled %s" % (change, labelDetails) files = p4CmdList("files %s...@%s" % (branchPrefix, change)) @@ -674,129 +700,9 @@ class P4Sync(Command): if not self.silent: print "Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change) - def extractFilesInCommitToBranch(self, files, branchPrefix): - newFiles = [] - - for file in files: - path = file["path"] - if path.startswith(branchPrefix): - newFiles.append(file) - - return newFiles - - def findBranchSourceHeuristic(self, files, branch, branchPrefix): - for file in files: - action = file["action"] - if action != "integrate" and action != "branch": - continue - path = file["path"] - rev = file["rev"] - depotPath = path + "#" + rev - - log = p4CmdList("filelog \"%s\"" % depotPath) - if len(log) != 1: - print "eek! I got confused by the filelog of %s" % depotPath - sys.exit(1); - - log = log[0] - if log["action0"] != action: - print "eek! wrong action in filelog for %s : found %s, expected %s" % (depotPath, log["action0"], action) - sys.exit(1); - - branchAction = log["how0,0"] - - if not branchAction.endswith(" from"): - continue # ignore for branching - # print "eek! file %s was not branched from but instead: %s" % (depotPath, branchAction) - # sys.exit(1); - - source = log["file0,0"] - if source.startswith(branchPrefix): - continue - - lastSourceRev = log["erev0,0"] - - sourceLog = p4CmdList("filelog -m 1 \"%s%s\"" % (source, lastSourceRev)) - if len(sourceLog) != 1: - print "eek! I got confused by the source filelog of %s%s" % (source, lastSourceRev) - sys.exit(1); - sourceLog = sourceLog[0] - - relPath = source[len(self.globalPrefix):] - # strip off the filename - relPath = relPath[0:relPath.rfind("/")] - - for candidate in self.knownBranches: - if self.isSubPathOf(relPath, candidate) and candidate != branch: - return candidate - - return "" - - def changeIsBranchMerge(self, sourceBranch, destinationBranch, change): - sourceFiles = {} - for file in p4CmdList("files %s...@%s" % (self.globalPrefix + sourceBranch + "/", change)): - if file["action"] == "delete": - continue - sourceFiles[file["depotFile"]] = file - - destinationFiles = {} - for file in p4CmdList("files %s...@%s" % (self.globalPrefix + destinationBranch + "/", change)): - destinationFiles[file["depotFile"]] = file - - for fileName in sourceFiles.keys(): - integrations = [] - deleted = False - integrationCount = 0 - for integration in p4CmdList("integrated \"%s\"" % fileName): - toFile = integration["fromFile"] # yes, it's true, it's fromFile - if not toFile in destinationFiles: - continue - destFile = destinationFiles[toFile] - if destFile["action"] == "delete": - # print "file %s has been deleted in %s" % (fileName, toFile) - deleted = True - break - integrationCount += 1 - if integration["how"] == "branch from": - continue - - if int(integration["change"]) == change: - integrations.append(integration) - continue - if int(integration["change"]) > change: - continue - - destRev = int(destFile["rev"]) - - startRev = integration["startFromRev"][1:] - if startRev == "none": - startRev = 0 - else: - startRev = int(startRev) - - endRev = integration["endFromRev"][1:] - if endRev == "none": - endRev = 0 - else: - endRev = int(endRev) - - initialBranch = (destRev == 1 and integration["how"] != "branch into") - inRange = (destRev >= startRev and destRev <= endRev) - newer = (destRev > startRev and destRev > endRev) - - if initialBranch or inRange or newer: - integrations.append(integration) - - if deleted: - continue - - if len(integrations) == 0 and integrationCount > 1: - print "file %s was not integrated from %s into %s" % (fileName, sourceBranch, destinationBranch) - return False - - return True - - def getUserMap(self): + def getUserMapFromPerforceServer(self): + if self.userMapFromPerforceServer: + return self.users = {} for output in p4CmdList("users"): @@ -804,18 +710,39 @@ class P4Sync(Command): continue self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" + cache = open(gitdir + "/p4-usercache.txt", "wb") + for user in self.users.keys(): + cache.write("%s\t%s\n" % (user, self.users[user])) + cache.close(); + self.userMapFromPerforceServer = True + + def loadUserMapFromCache(self): + self.users = {} + self.userMapFromPerforceServer = False + try: + cache = open(gitdir + "/p4-usercache.txt", "rb") + lines = cache.readlines() + cache.close() + for line in lines: + entry = line[:-1].split("\t") + self.users[entry[0]] = entry[1] + except IOError: + self.getUserMapFromPerforceServer() + def getLabels(self): self.labels = {} - l = p4CmdList("labels %s..." % self.globalPrefix) + l = p4CmdList("labels %s..." % self.depotPath) if len(l) > 0 and not self.silent: - print "Finding files belonging to labels in %s" % self.globalPrefix + print "Finding files belonging to labels in %s" % self.depotPath for output in l: label = output["label"] revisions = {} newestChange = 0 - for file in p4CmdList("files //...@%s" % label): + if self.verbose: + print "Querying files for label %s" % label + for file in p4CmdList("files %s...@%s" % (self.depotPath, label)): revisions[file["depotFile"]] = file["rev"] change = int(file["change"]) if change > newestChange: @@ -823,116 +750,229 @@ class P4Sync(Command): self.labels[newestChange] = [output, revisions] + if self.verbose: + print "Label changes: %s" % self.labels.keys() + + def getBranchMapping(self): + self.projectName = self.depotPath[self.depotPath[:-1].rfind("/") + 1:] + + for info in p4CmdList("branches"): + details = p4Cmd("branch -o %s" % info["branch"]) + viewIdx = 0 + while details.has_key("View%s" % viewIdx): + paths = details["View%s" % viewIdx].split(" ") + viewIdx = viewIdx + 1 + # require standard //depot/foo/... //depot/bar/... mapping + if len(paths) != 2 or not paths[0].endswith("/...") or not paths[1].endswith("/..."): + continue + source = paths[0] + destination = paths[1] + if source.startswith(self.depotPath) and destination.startswith(self.depotPath): + source = source[len(self.depotPath):-4] + destination = destination[len(self.depotPath):-4] + if destination not in self.knownBranches: + self.knownBranches[destination] = source + if source not in self.knownBranches: + self.knownBranches[source] = source + + def listExistingP4GitBranches(self): + self.p4BranchesInGit = [] + + cmdline = "git rev-parse --symbolic " + if self.importIntoRemotes: + cmdline += " --remotes" + else: + cmdline += " --branches" + + for line in mypopen(cmdline).readlines(): + if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD\n"): + continue + if self.importIntoRemotes: + # strip off p4 + branch = line[3:-1] + else: + branch = line[:-1] + self.p4BranchesInGit.append(branch) + self.initialParents[self.refPrefix + branch] = parseRevision(line[:-1]) + + def createOrUpdateBranchesFromOrigin(self): + if not self.silent: + print "Creating/updating branch(es) in %s based on origin branch(es)" % self.refPrefix + + for line in mypopen("git rev-parse --symbolic --remotes"): + if (not line.startswith("origin/")) or line.endswith("HEAD\n"): + continue + + headName = line[len("origin/"):-1] + remoteHead = self.refPrefix + headName + originHead = "origin/" + headName + + [originPreviousDepotPath, originP4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(originHead)) + if len(originPreviousDepotPath) == 0 or len(originP4Change) == 0: + continue + + update = False + if not gitBranchExists(remoteHead): + if self.verbose: + print "creating %s" % remoteHead + update = True + else: + [p4PreviousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(remoteHead)) + if len(p4Change) > 0: + if originPreviousDepotPath == p4PreviousDepotPath: + originP4Change = int(originP4Change) + p4Change = int(p4Change) + if originP4Change > p4Change: + print "%s (%s) is newer than %s (%s). Updating p4 branch from origin." % (originHead, originP4Change, remoteHead, p4Change) + update = True + else: + print "Ignoring: %s was imported from %s while %s was imported from %s" % (originHead, originPreviousDepotPath, remoteHead, p4PreviousDepotPath) + + if update: + system("git update-ref %s %s" % (remoteHead, originHead)) + def run(self, args): - self.globalPrefix = "" + self.depotPath = "" self.changeRange = "" self.initialParent = "" + self.previousDepotPath = "" + # map from branch depot path to parent branch + self.knownBranches = {} + self.initialParents = {} + + if self.importIntoRemotes: + self.refPrefix = "refs/remotes/p4/" + else: + self.refPrefix = "refs/heads/" + + if self.syncWithOrigin: + print "Syncing with origin first as requested by calling git fetch origin" + system("git fetch origin") + + createP4HeadRef = False; if len(self.branch) == 0: - self.branch = "p4" + self.branch = self.refPrefix + "master" + if gitBranchExists("refs/heads/p4") and self.importIntoRemotes: + system("git update-ref %s refs/heads/p4" % self.branch) + system("git branch -D p4"); + # create it /after/ importing, when master exists + if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: + createP4HeadRef = True if len(args) == 0: - if not gitBranchExists(self.branch) and gitBranchExists("origin"): + self.createOrUpdateBranchesFromOrigin() + self.listExistingP4GitBranches() + + if len(self.p4BranchesInGit) > 1: if not self.silent: - print "Creating %s branch in git repository based on origin" % self.branch - system("git branch %s origin" % self.branch) + print "Importing from/into multiple branches" + self.detectBranches = True + + if self.verbose: + print "branches: %s" % self.p4BranchesInGit + + p4Change = 0 + for branch in self.p4BranchesInGit: + depotPath, change = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.refPrefix + branch)) - [self.previousDepotPath, p4Change] = extractDepotPathAndChangeFromGitLog(extractLogMessageFromGitCommit(self.branch)) - if len(self.previousDepotPath) > 0 and len(p4Change) > 0: - p4Change = int(p4Change) + 1 - self.globalPrefix = self.previousDepotPath + if self.verbose: + print "path %s change %s" % (depotPath, change) + + if len(depotPath) > 0 and len(change) > 0: + change = int(change) + 1 + p4Change = max(p4Change, change) + + if len(self.previousDepotPath) == 0: + self.previousDepotPath = depotPath + else: + i = 0 + l = min(len(self.previousDepotPath), len(depotPath)) + while i < l and self.previousDepotPath[i] == depotPath[i]: + i = i + 1 + self.previousDepotPath = self.previousDepotPath[:i] + + if p4Change > 0: + self.depotPath = self.previousDepotPath self.changeRange = "@%s,#head" % p4Change - self.initialParent = self.branch - self.tagLastChange = False - if not self.silent: + self.initialParent = parseRevision(self.branch) + if not self.silent and not self.detectBranches: print "Performing incremental import into %s git branch" % self.branch - self.branch = "refs/heads/" + self.branch - - if len(self.globalPrefix) == 0: - self.globalPrefix = self.previousDepotPath = os.popen("git repo-config --get p4.depotpath").read() + if not self.branch.startswith("refs/"): + self.branch = "refs/heads/" + self.branch - if len(self.globalPrefix) != 0: - self.globalPrefix = self.globalPrefix[:-1] + if len(self.depotPath) != 0: + self.depotPath = self.depotPath[:-1] - if len(args) == 0 and len(self.globalPrefix) != 0: + if len(args) == 0 and len(self.depotPath) != 0: if not self.silent: - print "Depot path: %s" % self.globalPrefix + print "Depot path: %s" % self.depotPath elif len(args) != 1: return False else: - if len(self.globalPrefix) != 0 and self.globalPrefix != args[0]: - print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.globalPrefix, args[0]) + if len(self.depotPath) != 0 and self.depotPath != args[0]: + print "previous import used depot path %s and now %s was specified. this doesn't work!" % (self.depotPath, args[0]) sys.exit(1) - self.globalPrefix = args[0] + self.depotPath = args[0] self.revision = "" self.users = {} - self.lastChange = 0 - self.initialTag = "" - if self.globalPrefix.find("@") != -1: - atIdx = self.globalPrefix.index("@") - self.changeRange = self.globalPrefix[atIdx:] + if self.depotPath.find("@") != -1: + atIdx = self.depotPath.index("@") + self.changeRange = self.depotPath[atIdx:] if self.changeRange == "@all": self.changeRange = "" elif self.changeRange.find(",") == -1: self.revision = self.changeRange self.changeRange = "" - self.globalPrefix = self.globalPrefix[0:atIdx] - elif self.globalPrefix.find("#") != -1: - hashIdx = self.globalPrefix.index("#") - self.revision = self.globalPrefix[hashIdx:] - self.globalPrefix = self.globalPrefix[0:hashIdx] + self.depotPath = self.depotPath[0:atIdx] + elif self.depotPath.find("#") != -1: + hashIdx = self.depotPath.index("#") + self.revision = self.depotPath[hashIdx:] + self.depotPath = self.depotPath[0:hashIdx] elif len(self.previousDepotPath) == 0: self.revision = "#head" - if self.globalPrefix.endswith("..."): - self.globalPrefix = self.globalPrefix[:-3] + if self.depotPath.endswith("..."): + self.depotPath = self.depotPath[:-3] - if not self.globalPrefix.endswith("/"): - self.globalPrefix += "/" + if not self.depotPath.endswith("/"): + self.depotPath += "/" - self.getUserMap() + self.loadUserMapFromCache() self.labels = {} if self.detectLabels: self.getLabels(); - if len(self.changeRange) == 0: - try: - sout, sin, serr = popen2.popen3("git name-rev --tags `git rev-parse %s`" % self.branch) - output = sout.read() - if output.endswith("\n"): - output = output[:-1] - tagIdx = output.index(" tags/p4/") - caretIdx = output.find("^") - endPos = len(output) - if caretIdx != -1: - endPos = caretIdx - self.rev = int(output[tagIdx + 9 : endPos]) + 1 - self.changeRange = "@%s,#head" % self.rev - self.initialParent = os.popen("git rev-parse %s" % self.branch).read()[:-1] - self.initialTag = "p4/%s" % (int(self.rev) - 1) - except: - pass - - self.tz = - time.timezone / 36 - tzsign = ("%s" % self.tz)[0] - if tzsign != '+' and tzsign != '-': - self.tz = "+" + ("%s" % self.tz) - - self.gitOutput, self.gitStream, self.gitError = popen2.popen3("git fast-import") + if self.detectBranches: + self.getBranchMapping(); + if self.verbose: + print "p4-git branches: %s" % self.p4BranchesInGit + print "initial parents: %s" % self.initialParents + for b in self.p4BranchesInGit: + if b != "master": + b = b[len(self.projectName):] + self.createdBranches.add(b) + + self.tz = "%+03d%02d" % (- time.timezone / 3600, ((- time.timezone % 3600) / 60)) + + importProcess = subprocess.Popen(["git", "fast-import"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE); + self.gitOutput = importProcess.stdout + self.gitStream = importProcess.stdin + self.gitError = importProcess.stderr if len(self.revision) > 0: - print "Doing initial import of %s from revision %s" % (self.globalPrefix, self.revision) + print "Doing initial import of %s from revision %s" % (self.depotPath, self.revision) details = { "user" : "git perforce import user", "time" : int(time.time()) } - details["desc"] = "Initial import of %s from the state at revision %s" % (self.globalPrefix, self.revision) + details["desc"] = "Initial import of %s from the state at revision %s" % (self.depotPath, self.revision) details["change"] = self.revision newestRevision = 0 fileCnt = 0 - for info in p4CmdList("files %s...%s" % (self.globalPrefix, self.revision)): + for info in p4CmdList("files %s...%s" % (self.depotPath, self.revision)): change = int(info["change"]) if change > newestRevision: newestRevision = change @@ -950,8 +990,9 @@ class P4Sync(Command): details["change"] = newestRevision try: - self.commit(details, self.extractFilesFromCommit(details), self.branch, self.globalPrefix) + self.commit(details, self.extractFilesFromCommit(details), self.branch, self.depotPath) except IOError: + print "IO error with git fast-import. Is your git version recent enough?" print self.gitError.read() else: @@ -968,7 +1009,9 @@ class P4Sync(Command): changes.sort() else: - output = os.popen("p4 changes %s...%s" % (self.globalPrefix, self.changeRange)).readlines() + if self.verbose: + print "Getting p4 changes for %s...%s" % (self.depotPath, self.changeRange) + output = mypopen("p4 changes %s...%s" % (self.depotPath, self.changeRange)).readlines() for line in output: changeNum = line.split(" ")[1] @@ -976,89 +1019,112 @@ class P4Sync(Command): changes.reverse() + if len(self.maxChanges) > 0: + changes = changes[0:min(int(self.maxChanges), len(changes))] + if len(changes) == 0: if not self.silent: - print "no changes to import!" + print "No changes to import!" return True + self.updatedBranches = set() + cnt = 1 for change in changes: description = p4Cmd("describe %s" % change) if not self.silent: - sys.stdout.write("\rimporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) + sys.stdout.write("\rImporting revision %s (%s%%)" % (change, cnt * 100 / len(changes))) sys.stdout.flush() cnt = cnt + 1 try: - files = self.extractFilesFromCommit(description) if self.detectBranches: - for branch in self.branchesForCommit(files): - self.knownBranches.add(branch) - branchPrefix = self.globalPrefix + branch + "/" + branches = self.splitFilesIntoBranches(description) + for branch in branches.keys(): + branchPrefix = self.depotPath + branch + "/" - filesForCommit = self.extractFilesInCommitToBranch(files, branchPrefix) - - merged = "" parent = "" - ########### remove cnt!!! - if branch not in self.createdBranches and cnt > 2: + + filesForCommit = branches[branch] + + if self.verbose: + print "branch is %s" % branch + + self.updatedBranches.add(branch) + + if branch not in self.createdBranches: self.createdBranches.add(branch) - parent = self.findBranchParent(branchPrefix, files) + parent = self.knownBranches[branch] if parent == branch: parent = "" - # elif len(parent) > 0: - # print "%s branched off of %s" % (branch, parent) + elif self.verbose: + print "parent determined through known branches: %s" % parent - if len(parent) == 0: - merged = self.findBranchSourceHeuristic(filesForCommit, branch, branchPrefix) - if len(merged) > 0: - print "change %s could be a merge from %s into %s" % (description["change"], merged, branch) - if not self.changeIsBranchMerge(merged, branch, int(description["change"])): - merged = "" + # main branch? use master + if branch == "main": + branch = "master" + else: + branch = self.projectName + branch - branch = "refs/heads/" + branch + if parent == "main": + parent = "master" + elif len(parent) > 0: + parent = self.projectName + parent + + branch = self.refPrefix + branch if len(parent) > 0: - parent = "refs/heads/" + parent - if len(merged) > 0: - merged = "refs/heads/" + merged - self.commit(description, files, branch, branchPrefix, parent, merged) + parent = self.refPrefix + parent + + if self.verbose: + print "looking for initial parent for %s; current parent is %s" % (branch, parent) + + if len(parent) == 0 and branch in self.initialParents: + parent = self.initialParents[branch] + del self.initialParents[branch] + + self.commit(description, filesForCommit, branch, branchPrefix, parent) else: - self.commit(description, files, self.branch, self.globalPrefix, self.initialParent) + files = self.extractFilesFromCommit(description) + self.commit(description, files, self.branch, self.depotPath, self.initialParent) self.initialParent = "" except IOError: print self.gitError.read() sys.exit(1) - if not self.silent: - print "" - - if self.tagLastChange: - self.gitStream.write("reset refs/tags/p4/%s\n" % self.lastChange) - self.gitStream.write("from %s\n\n" % self.branch); + if not self.silent: + print "" + if len(self.updatedBranches) > 0: + sys.stdout.write("Updated branches: ") + for b in self.updatedBranches: + sys.stdout.write("%s " % b) + sys.stdout.write("\n") self.gitStream.close() + if importProcess.wait() != 0: + die("fast-import failed: %s" % self.gitError.read()) self.gitOutput.close() self.gitError.close() - os.popen("git repo-config p4.depotpath %s" % self.globalPrefix).read() - if len(self.initialTag) > 0: - os.popen("git tag -d %s" % self.initialTag).read() + if createP4HeadRef: + system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) return True class P4Rebase(Command): def __init__(self): Command.__init__(self) - self.options = [ ] + self.options = [ optparse.make_option("--with-origin", dest="syncWithOrigin", action="store_true") ] self.description = "Fetches the latest revision from perforce and rebases the current work (branch) against it" + self.syncWithOrigin = False def run(self, args): sync = P4Sync() + sync.syncWithOrigin = self.syncWithOrigin sync.run([]) print "Rebasing the current branch" - oldHead = os.popen("git rev-parse HEAD").read()[:-1] + oldHead = mypopen("git rev-parse HEAD").read()[:-1] system("git rebase p4") system("git diff-tree --stat --summary -M %s HEAD" % oldHead) return True @@ -1069,9 +1135,10 @@ class P4Clone(P4Sync): self.description = "Creates a new git repository and imports from Perforce into it" self.usage = "usage: %prog [options] //depot/path[@revRange] [directory]" self.needsGit = False - self.tagLastChange = False def run(self, args): + global gitdir + if len(args) < 1: return False depotPath = args[0] @@ -1107,12 +1174,15 @@ class P4Clone(P4Sync): os.makedirs(dir) os.chdir(dir) system("git init") + gitdir = os.getcwd() + "/.git" if not P4Sync.run(self, [depotPath]): return False - os.wait() if self.branch != "master": - system("git branch master p4") - system("git checkout -f") + if gitBranchExists("refs/remotes/p4/master"): + system("git branch master refs/remotes/p4/master") + system("git checkout -f") + else: + print "Could not detect main branch. No checkout/master branch created." return True class HelpFormatter(optparse.IndentedHelpFormatter): @@ -1135,11 +1205,11 @@ def printUsage(commands): commands = { "debug" : P4Debug(), - "clean-tags" : P4CleanTags(), "submit" : P4Submit(), "sync" : P4Sync(), "rebase" : P4Rebase(), - "clone" : P4Clone() + "clone" : P4Clone(), + "rollback" : P4RollBack() } if len(sys.argv[1:]) == 0: @@ -1176,9 +1246,11 @@ if cmd.needsGit: if len(gitdir) == 0: gitdir = ".git" if not isValidGitDir(gitdir): - cdup = os.popen("git rev-parse --show-cdup").read()[:-1] - if isValidGitDir(cdup + "/" + gitdir): - os.chdir(cdup) + gitdir = mypopen("git rev-parse --git-dir").read()[:-1] + if os.path.exists(gitdir): + cdup = mypopen("git rev-parse --show-cdup").read()[:-1]; + if len(cdup) > 0: + os.chdir(cdup); if not isValidGitDir(gitdir): if isValidGitDir(gitdir + "/.git"):