X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=contrib%2Ffast-import%2Fgit-p4;h=e3404ca853c8b54f1a0c64be7038d282746652a7;hb=144ff46b196e49fd52b2ecf0aaa1db4c190393b9;hp=01efd92809acab951d550fd69776912174f56eb5;hpb=a3287be5bc40c1aa036eb5422db5a6a087d1736e;p=git.git diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 01efd9280..e3404ca85 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -16,40 +16,44 @@ from sets import Set; verbose = False +def die(msg): + if verbose: + raise Exception(msg) + else: + sys.stderr.write(msg + "\n") + sys.exit(1) + def write_pipe(c, str): if verbose: - sys.stderr.write('writing pipe: %s\n' % c) + sys.stderr.write('Writing pipe: %s\n' % c) pipe = os.popen(c, 'w') val = pipe.write(str) if pipe.close(): - sys.stderr.write('Command failed: %s' % c) - sys.exit(1) + die('Command failed: %s' % c) return val def read_pipe(c, ignore_error=False): if verbose: - sys.stderr.write('reading pipe: %s\n' % c) + sys.stderr.write('Reading pipe: %s\n' % c) pipe = os.popen(c, 'rb') val = pipe.read() if pipe.close() and not ignore_error: - sys.stderr.write('Command failed: %s\n' % c) - sys.exit(1) + die('Command failed: %s' % c) return val def read_pipe_lines(c): if verbose: - sys.stderr.write('reading pipe: %s\n' % c) + sys.stderr.write('Reading pipe: %s\n' % c) ## todo: check return status pipe = os.popen(c, 'rb') val = pipe.readlines() if pipe.close(): - sys.stderr.write('Command failed: %s\n' % c) - sys.exit(1) + die('Command failed: %s' % c) return val @@ -59,21 +63,34 @@ def system(cmd): if os.system(cmd) != 0: die("command failed: %s" % cmd) -def p4CmdList(cmd): +def p4CmdList(cmd, stdin=None, stdin_mode='w+b'): cmd = "p4 -G %s" % cmd if verbose: sys.stderr.write("Opening pipe: %s\n" % cmd) - pipe = os.popen(cmd, "rb") + + # Use a temporary file to avoid deadlocks without + # subprocess.communicate(), which would put another copy + # of stdout into memory. + stdin_file = None + if stdin is not None: + stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) + stdin_file.write(stdin) + stdin_file.flush() + stdin_file.seek(0) + + p4 = subprocess.Popen(cmd, shell=True, + stdin=stdin_file, + stdout=subprocess.PIPE) result = [] try: while True: - entry = marshal.load(pipe) + entry = marshal.load(p4.stdout) result.append(entry) except EOFError: pass - exitCode = pipe.close() - if exitCode != None: + exitCode = p4.wait() + if exitCode != 0: entry = {} entry["p4ExitCode"] = exitCode result.append(entry) @@ -105,10 +122,6 @@ def p4Where(depotPath): clientPath = clientPath[:-3] return clientPath -def die(msg): - sys.stderr.write(msg + "\n") - sys.exit(1) - def currentGitBranch(): return read_pipe("git name-rev HEAD").split(" ")[1].strip() @@ -153,7 +166,11 @@ def extractSettingsGitLog(log): values[key] = val - values['depot-paths'] = values.get("depot-paths").split(',') + paths = values.get("depot-paths") + if not paths: + paths = values.get("depot-path") + if paths: + values['depot-paths'] = paths.split(',') return values def gitBranchExists(branch): @@ -164,6 +181,56 @@ def gitBranchExists(branch): def gitConfig(key): return read_pipe("git config %s" % key, ignore_error=True).strip() +def p4BranchesInGit(branchesAreInRemotes = True): + branches = {} + + cmdline = "git rev-parse --symbolic " + if branchesAreInRemotes: + cmdline += " --remotes" + else: + cmdline += " --branches" + + for line in read_pipe_lines(cmdline): + line = line.strip() + + ## only import to p4/ + if not line.startswith('p4/') or line == "p4/HEAD": + continue + branch = line + + # strip off p4 + branch = re.sub ("^p4/", "", line) + + branches[branch] = parseRevision(line) + return branches + +def findUpstreamBranchPoint(head = "HEAD"): + branches = p4BranchesInGit() + # map from depot-path to branch name + branchByDepotPath = {} + for branch in branches.keys(): + tip = branches[branch] + log = extractLogMessageFromGitCommit(tip) + settings = extractSettingsGitLog(log) + if settings.has_key("depot-paths"): + paths = ",".join(settings["depot-paths"]) + branchByDepotPath[paths] = "remotes/p4/" + branch + + settings = None + parent = 0 + while parent < 65535: + commit = head + "~%s" % parent + log = extractLogMessageFromGitCommit(commit) + settings = extractSettingsGitLog(log) + if settings.has_key("depot-paths"): + paths = ",".join(settings["depot-paths"]) + if branchByDepotPath.has_key(paths): + return [branchByDepotPath[paths], settings] + + parent = parent + 1 + + return ["", settings] + class Command: def __init__(self): self.usage = "usage: %prog [options]" @@ -173,13 +240,18 @@ class P4Debug(Command): def __init__(self): Command.__init__(self) self.options = [ - optparse.make_option("--verbose", dest="verbose", action="store_true"), + optparse.make_option("--verbose", dest="verbose", action="store_true", + default=False), ] self.description = "A tool to debug the output of p4 -G." self.needsGit = False + self.verbose = False def run(self, args): + j = 0 for output in p4CmdList(" ".join(args)): + print 'Element: %d' % j + j += 1 print output return True @@ -268,6 +340,8 @@ class P4Submit(Command): self.origin = "" self.directSubmit = False self.trustMeLikeAFool = False + self.verbose = False + self.isWindows = (platform.system() == "Windows") self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" @@ -380,15 +454,17 @@ class P4Submit(Command): system(applyPatchCmd) for f in filesToAdd: - system("p4 add %s" % f) + system("p4 add \"%s\"" % f) for f in filesToDelete: - system("p4 revert %s" % f) - system("p4 delete %s" % f) + system("p4 revert \"%s\"" % f) + system("p4 delete \"%s\"" % f) logMessage = "" if not self.directSubmit: logMessage = extractLogMessageFromGitCommit(id) logMessage = logMessage.replace("\n", "\n\t") + if self.isWindows: + logMessage = logMessage.replace("\n", "\r\n") logMessage = logMessage.strip() template = read_pipe("p4 change -o") @@ -435,6 +511,8 @@ class P4Submit(Command): tmpFile.close() os.remove(fileName) submitTemplate = message[:message.index(separatorLine)] + if self.isWindows: + submitTemplate = submitTemplate.replace("\r\n", "\n") if response == "y" or response == "yes": if self.dryRun: @@ -470,9 +548,6 @@ class P4Submit(Command): % (fileName, fileName)) def run(self, args): - # make gitdir absolute so we can cd out into the perforce checkout - os.environ["GIT_DIR"] = gitdir - if len(args) == 0: self.master = currentGitBranch() if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master): @@ -482,13 +557,13 @@ class P4Submit(Command): else: return False - depotPath = "" - settings = None - if gitBranchExists("p4"): - settings = extractSettingsGitLog(extractLogMessageFromGitCommit("p4")) - if len(depotPath) == 0 and gitBranchExists("origin"): - settings = extractSettingsGitLog(extractLogMessageFromGitCommit("origin")) - depotPaths = settings['depot-paths'] + [upstream, settings] = findUpstreamBranchPoint() + depotPath = settings['depot-paths'][0] + if len(self.origin) == 0: + self.origin = upstream + + if self.verbose: + print "Origin branch is " + self.origin if len(depotPath) == 0: print "Internal error: cannot locate perforce depot path from existing branches" @@ -519,12 +594,6 @@ class P4Submit(Command): if response == "y" or response == "yes": system("p4 sync ...") - if len(self.origin) == 0: - if gitBranchExists("p4"): - self.origin = "p4" - else: - self.origin = "origin" - if self.reset: self.firstTime = True @@ -580,9 +649,11 @@ class P4Sync(Command): optparse.make_option("--silent", dest="silent", action="store_true"), optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), optparse.make_option("--verbose", dest="verbose", action="store_true"), - optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"), + optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false", + help="Import into refs/heads/ , not refs/remotes"), optparse.make_option("--max-changes", dest="maxChanges"), - optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true') + optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true', + help="Keep entire BRANCH/DIR/SUBDIR prefix during import") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -607,6 +678,7 @@ class P4Sync(Command): self.isWindows = (platform.system() == "Windows") self.keepRepoPath = False self.depotPaths = None + self.p4BranchesInGit = [] if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False @@ -669,41 +741,47 @@ class P4Sync(Command): if branch not in branches: branches[branch] = [] branches[branch].append(file) + break return branches ## Should move this out, doesn't use SELF. def readP4Files(self, files): - specs = [(f['path'] + "#" + f['rev'], f) for f in files] - - data = read_pipe('p4 print %s' % ' '.join(['"%s"' % spec - for (spec, info) in specs])) + files = [f for f in files + if f['action'] != 'delete'] - idx = 0 - for j in range(0, len(specs)): - filespec, info = specs[j] - - assert idx < len(data) - if data[idx:idx + len(filespec)] != filespec: - assert False - idx = data.find ('\n', idx) - assert idx > 0 - idx += 1 - - start = idx + if not files: + return - end = -1 - if j < len(specs)-1: - next_spec, next_info = specs[j+1] - end = data.find(next_spec, start) + filedata = p4CmdList('-x - print', + stdin='\n'.join(['%s#%s' % (f['path'], f['rev']) + for f in files]), + stdin_mode='w+') + if "p4ExitCode" in filedata[0]: + die("Problems executing p4. Error: [%d]." + % (filedata[0]['p4ExitCode'])); + + j = 0; + contents = {} + while j < len(filedata): + stat = filedata[j] + j += 1 + text = '' + while j < len(filedata) and filedata[j]['code'] in ('text', + 'binary'): + text += filedata[j]['data'] + j += 1 + + + if not stat.has_key('depotFile'): + sys.stderr.write("p4 print fails with: %s\n" % repr(stat)) + continue - assert end >= 0 - else: - end = len(data) + contents[stat['depotFile']] = text - info['data'] = data[start:end] - idx = end - assert idx == len(data) + for f in files: + assert not f.has_key('data') + f['data'] = contents[f['path']] def commit(self, details, files, branch, branchPrefixes, parent = ""): epoch = details["time"] @@ -712,6 +790,20 @@ class P4Sync(Command): if self.verbose: print "commit into %s" % branch + # start with reading files; if that fails, we should not + # create a commit. + new_files = [] + for f in files: + if [p for p in branchPrefixes if f['path'].startswith(p)]: + new_files.append (f) + else: + sys.stderr.write("Ignoring file outside of prefix: %s\n" % path) + files = new_files + self.readP4Files(files) + + + + self.gitStream.write("commit %s\n" % branch) # gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) @@ -727,28 +819,17 @@ class P4Sync(Command): self.gitStream.write("data < 0: + self.gitStream.write(": options = %s" % details['options']) + self.gitStream.write("]\nEOT\n\n") if len(parent) > 0: if self.verbose: print "parent %s" % parent self.gitStream.write("from %s\n" % parent) - - new_files = [] - for f in files: - if [p for p in branchPrefixes if f['path'].startswith(p)]: - new_files.append (f) - else: - sys.stderr.write("Ignoring file outside of prefix: %s\n" % path) - files = new_files - - self.readP4Files(files) for file in files: if file["type"] == "apple": print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path'] @@ -767,7 +848,7 @@ class P4Sync(Command): if self.isWindows and file["type"].endswith("text"): data = data.replace("\r\n", "\n") - self.gitStream.write("M %s inline %s\n" % (mode, relPath)) + self.gitStream.write("M %d inline %s\n" % (mode, relPath)) self.gitStream.write("data %s\n" % len(data)) self.gitStream.write(data) self.gitStream.write("\n") @@ -879,10 +960,17 @@ class P4Sync(Command): if self.verbose: print "Label changes: %s" % self.labels.keys() - def getBranchMapping(self): + def guessProjectName(self): + for p in self.depotPaths: + if p.endswith("/"): + p = p[:-1] + p = p[p.strip().rfind("/") + 1:] + if not p.endswith("/"): + p += "/" + return p - ## FIXME - what's a P4 projectName ? - self.projectName = self.depotPath[self.depotPath.strip().rfind("/") + 1:] + def getBranchMapping(self): + lostAndFoundBranches = set() for info in p4CmdList("branches"): details = p4Cmd("branch -o %s" % info["branch"]) @@ -895,47 +983,50 @@ class P4Sync(Command): continue source = paths[0] destination = paths[1] - if source.startswith(self.depotPath) and destination.startswith(self.depotPath): - source = source[len(self.depotPath):-4] - destination = destination[len(self.depotPath):-4] - if destination not in self.knownBranches: - self.knownBranches[destination] = source - if source not in self.knownBranches: - self.knownBranches[source] = source + ## HACK + if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]): + source = source[len(self.depotPaths[0]):-4] + destination = destination[len(self.depotPaths[0]):-4] + + if destination in self.knownBranches: + if not self.silent: + print "p4 branch %s defines a mapping from %s to %s" % (info["branch"], source, destination) + print "but there exists another mapping from %s to %s already!" % (self.knownBranches[destination], destination) + continue - def listExistingP4GitBranches(self): - self.p4BranchesInGit = [] + self.knownBranches[destination] = source - cmdline = "git rev-parse --symbolic " - if self.importIntoRemotes: - cmdline += " --remotes" - else: - cmdline += " --branches" + lostAndFoundBranches.discard(destination) - for line in read_pipe_lines(cmdline): - line = line.strip() - if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD"): - continue + if source not in self.knownBranches: + lostAndFoundBranches.add(source) - if self.importIntoRemotes: - # strip off p4 - branch = re.sub ("^p4/", "", line) - self.p4BranchesInGit.append(branch) - self.initialParents[self.refPrefix + branch] = parseRevision(line) + for branch in lostAndFoundBranches: + self.knownBranches[branch] = branch + + def listExistingP4GitBranches(self): + # branches holds mapping from name to commit + branches = p4BranchesInGit(self.importIntoRemotes) + self.p4BranchesInGit = branches.keys() + for branch in branches.keys(): + self.initialParents[self.refPrefix + branch] = branches[branch] def createOrUpdateBranchesFromOrigin(self): if not self.silent: - print "Creating/updating branch(es) in %s based on origin branch(es)" % self.refPrefix + print ("Creating/updating branch(es) in %s based on origin branch(es)" + % self.refPrefix) + + originPrefix = "origin/p4/" for line in read_pipe_lines("git rev-parse --symbolic --remotes"): line = line.strip() - if (not line.startswith("origin/")) or line.endswith("HEAD\n"): + if (not line.startswith(originPrefix)) or line.endswith("HEAD"): continue - headName = line[len("origin/"):] + headName = line[len(originPrefix):] remoteHead = self.refPrefix + headName - originHead = "origin/" + headName + originHead = line original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) if (not original.has_key('depot-paths') @@ -948,7 +1039,7 @@ class P4Sync(Command): print "creating %s" % remoteHead update = True else: - settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) + settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) if settings.has_key('change') > 0: if settings['depot-paths'] == original['depot-paths']: originP4Change = int(original['change']) @@ -988,12 +1079,14 @@ class P4Sync(Command): # map from branch depot path to parent branch self.knownBranches = {} self.initialParents = {} - self.hasOrigin = gitBranchExists("origin") + self.hasOrigin = gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") + if not self.syncWithOrigin: + self.hasOrigin = False if self.importIntoRemotes: self.refPrefix = "refs/remotes/p4/" else: - self.refPrefix = "refs/heads/" + self.refPrefix = "refs/heads/p4/" if self.syncWithOrigin and self.hasOrigin: if not self.silent: @@ -1030,9 +1123,6 @@ class P4Sync(Command): settings = extractSettingsGitLog(logMsg) - if self.verbose: - print "path %s change %s" % (','.join(depotPaths), change) - self.readOptions(settings) if (settings.has_key('depot-paths') and settings.has_key ('change')): @@ -1045,18 +1135,20 @@ class P4Sync(Command): else: paths = [] for (prev, cur) in zip(self.previousDepotPaths, depotPaths): - for i in range(0, max(len(cur), len(prev))): + for i in range(0, min(len(cur), len(prev))): if cur[i] <> prev[i]: + i = i - 1 break - paths.append (cur[:i]) + paths.append (cur[:i + 1]) self.previousDepotPaths = paths if p4Change > 0: self.depotPaths = sorted(self.previousDepotPaths) self.changeRange = "@%s,#head" % p4Change - self.initialParent = parseRevision(self.branch) + if not self.detectBranches: + self.initialParent = parseRevision(self.branch) if not self.silent and not self.detectBranches: print "Performing incremental import into %s git branch" % self.branch @@ -1111,7 +1203,11 @@ class P4Sync(Command): self.getLabels(); if self.detectBranches: - self.getBranchMapping(); + ## FIXME - what's a P4 projectName ? + self.projectName = self.guessProjectName() + + if not self.hasOrigin: + self.getBranchMapping(); if self.verbose: print "p4-git branches: %s" % self.p4BranchesInGit print "initial parents: %s" % self.initialParents @@ -1131,8 +1227,8 @@ class P4Sync(Command): self.gitStream = importProcess.stdin self.gitError = importProcess.stderr - if len(self.revision) > 0: - print "Doing initial import of %s from revision %s" % (' '.join(self.depotPaths), self.revision) + if self.revision: + print "Doing initial import of %s from revision %s into %s" % (' '.join(self.depotPaths), self.revision, self.branch) details = { "user" : "git perforce import user", "time" : int(time.time()) } details["desc"] = ("Initial import of %s from the state at revision %s" @@ -1145,6 +1241,13 @@ class P4Sync(Command): + ' '.join(["%s...%s" % (p, self.revision) for p in self.depotPaths])): + + if info['code'] == 'error': + sys.stderr.write("p4 returned an error: %s\n" + % info['data']) + sys.exit(1) + + change = int(info["change"]) if change > newestRevision: newestRevision = change @@ -1154,7 +1257,7 @@ class P4Sync(Command): #fileCnt = fileCnt + 1 continue - for prop in [ "depotFile", "rev", "action", "type" ]: + for prop in ["depotFile", "rev", "action", "type" ]: details["%s%s" % (prop, fileCnt)] = info[prop] fileCnt = fileCnt + 1 @@ -1182,7 +1285,7 @@ class P4Sync(Command): changes.sort() else: if self.verbose: - print "Getting p4 changes for %s...%s" % (`self.depotPaths`, + print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths), self.changeRange) assert self.depotPaths output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, self.changeRange) @@ -1202,6 +1305,9 @@ class P4Sync(Command): print "No changes to import!" return True + if not self.silent and not self.detectBranches: + print "Import destination: %s" % self.branch + self.updatedBranches = set() cnt = 1 @@ -1263,7 +1369,7 @@ class P4Sync(Command): parent = self.initialParents[branch] del self.initialParents[branch] - self.commit(description, filesForCommit, branch, branchPrefix, parent) + self.commit(description, filesForCommit, branch, [branchPrefix], parent) else: files = self.extractFilesFromCommit(description) self.commit(description, files, self.branch, self.depotPaths, @@ -1296,13 +1402,22 @@ class P4Rebase(Command): self.options = [ ] self.description = ("Fetches the latest revision from perforce and " + "rebases the current work (branch) against it") + self.verbose = False def run(self, args): sync = P4Sync() sync.run([]) - print "Rebasing the current branch" + + [upstream, settings] = findUpstreamBranchPoint() + if len(upstream) == 0: + die("Cannot find upstream branchpoint for rebase") + + # the branchpoint may be p4/foo~3, so strip off the parent + upstream = re.sub("~[0-9]+$", "", upstream) + + print "Rebasing the current branch onto %s" % upstream oldHead = read_pipe("git rev-parse HEAD").strip() - system("git rebase p4") + system("git rebase %s" % upstream) system("git diff-tree --stat --summary -M %s HEAD" % oldHead) return True @@ -1336,15 +1451,21 @@ class P4Clone(P4Sync): sys.exit(1) depotPaths = args + + if not self.cloneDestination and len(depotPaths) > 1: + self.cloneDestination = depotPaths[-1] + depotPaths = depotPaths[:-1] + for p in depotPaths: if not p.startswith("//"): return False if not self.cloneDestination: - self.cloneDestination = self.defaultDestination() + self.cloneDestination = self.defaultDestination(args) - print "Importing from %s into %s" % (`depotPaths`, self.cloneDestination) - os.makedirs(self.cloneDestination) + print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination) + if not os.path.exists(self.cloneDestination): + os.makedirs(self.cloneDestination) os.chdir(self.cloneDestination) system("git init") self.gitdir = os.getcwd() + "/.git" @@ -1356,6 +1477,32 @@ class P4Clone(P4Sync): system("git checkout -f") else: print "Could not detect main branch. No checkout/master branch created." + + return True + +class P4Branches(Command): + def __init__(self): + Command.__init__(self) + self.options = [ ] + self.description = ("Shows the git branches that hold imports and their " + + "corresponding perforce depot paths") + self.verbose = False + + def run(self, args): + cmdline = "git rev-parse --symbolic " + cmdline += " --remotes" + + for line in read_pipe_lines(cmdline): + line = line.strip() + + if not line.startswith('p4/') or line == "p4/HEAD": + continue + branch = line + + log = extractLogMessageFromGitCommit("refs/remotes/%s" % branch) + settings = extractSettingsGitLog(log) + + print "%s <= %s (%s)" % (branch, ",".join(settings["depot-paths"]), settings["change"]) return True class HelpFormatter(optparse.IndentedHelpFormatter): @@ -1382,7 +1529,8 @@ commands = { "sync" : P4Sync, "rebase" : P4Rebase, "clone" : P4Clone, - "rollback" : P4RollBack + "rollback" : P4RollBack, + "branches" : P4Branches }