X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=contrib%2Ffast-import%2Fgit-p4;h=d03ba0b6ad4c570e33d45538ee9e3ec84d18c4ef;hb=7aded26ce870ba2998e276604fc3c18dad6133bd;hp=b280e9774227100015cb3c6c8ae7448d0ce844f0;hpb=bb6e09b27afeaae780dabfda7a07d59fc7efc4cf;p=git.git diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index b280e9774..d03ba0b6a 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -14,43 +14,46 @@ import re from sets import Set; -gitdir = os.environ.get("GIT_DIR", "") verbose = False +def die(msg): + if verbose: + raise Exception(msg) + else: + sys.stderr.write(msg + "\n") + sys.exit(1) + def write_pipe(c, str): if verbose: - sys.stderr.write('writing pipe: %s\n' % c) + sys.stderr.write('Writing pipe: %s\n' % c) pipe = os.popen(c, 'w') val = pipe.write(str) if pipe.close(): - sys.stderr.write('Command %s failed\n' % c) - sys.exit(1) + die('Command failed: %s' % c) return val def read_pipe(c, ignore_error=False): if verbose: - sys.stderr.write('reading pipe: %s\n' % c) + sys.stderr.write('Reading pipe: %s\n' % c) pipe = os.popen(c, 'rb') val = pipe.read() if pipe.close() and not ignore_error: - sys.stderr.write('Command %s failed\n' % c) - sys.exit(1) + die('Command failed: %s' % c) return val def read_pipe_lines(c): if verbose: - sys.stderr.write('reading pipe: %s\n' % c) + sys.stderr.write('Reading pipe: %s\n' % c) ## todo: check return status pipe = os.popen(c, 'rb') val = pipe.readlines() if pipe.close(): - sys.stderr.write('Command %s failed\n' % c) - sys.exit(1) + die('Command failed: %s' % c) return val @@ -62,6 +65,8 @@ def system(cmd): def p4CmdList(cmd): cmd = "p4 -G %s" % cmd + if verbose: + sys.stderr.write("Opening pipe: %s\n" % cmd) pipe = os.popen(cmd, "rb") result = [] @@ -104,10 +109,6 @@ def p4Where(depotPath): clientPath = clientPath[:-3] return clientPath -def die(msg): - sys.stderr.write(msg + "\n") - sys.exit(1) - def currentGitBranch(): return read_pipe("git name-rev HEAD").split(" ")[1].strip() @@ -152,7 +153,11 @@ def extractSettingsGitLog(log): values[key] = val - values['depot-paths'] = values.get("depot-paths").split(',') + paths = values.get("depot-paths") + if not paths: + paths = values.get("depot-path") + if paths: + values['depot-paths'] = paths.split(',') return values def gitBranchExists(branch): @@ -172,13 +177,18 @@ class P4Debug(Command): def __init__(self): Command.__init__(self) self.options = [ - optparse.make_option("--verbose", dest="verbose", action="store_true"), + optparse.make_option("--verbose", dest="verbose", action="store_true", + default=False), ] self.description = "A tool to debug the output of p4 -G." self.needsGit = False + self.verbose = False def run(self, args): + j = 0 for output in p4CmdList(" ".join(args)): + print 'Element: %d' % j + j += 1 print output return True @@ -267,6 +277,8 @@ class P4Submit(Command): self.origin = "" self.directSubmit = False self.trustMeLikeAFool = False + self.verbose = False + self.isWindows = (platform.system() == "Windows") self.logSubstitutions = {} self.logSubstitutions[""] = "%log%" @@ -388,6 +400,8 @@ class P4Submit(Command): if not self.directSubmit: logMessage = extractLogMessageFromGitCommit(id) logMessage = logMessage.replace("\n", "\n\t") + if self.isWindows: + logMessage = logMessage.replace("\n", "\r\n") logMessage = logMessage.strip() template = read_pipe("p4 change -o") @@ -434,6 +448,8 @@ class P4Submit(Command): tmpFile.close() os.remove(fileName) submitTemplate = message[:message.index(separatorLine)] + if self.isWindows: + submitTemplate = submitTemplate.replace("\r\n", "\n") if response == "y" or response == "yes": if self.dryRun: @@ -469,11 +485,6 @@ class P4Submit(Command): % (fileName, fileName)) def run(self, args): - global gitdir - # make gitdir absolute so we can cd out into the perforce checkout - gitdir = os.path.abspath(gitdir) - os.environ["GIT_DIR"] = gitdir - if len(args) == 0: self.master = currentGitBranch() if len(self.master) == 0 or not gitBranchExists("refs/heads/%s" % self.master): @@ -484,12 +495,27 @@ class P4Submit(Command): return False depotPath = "" - settings = None - if gitBranchExists("p4"): - settings = extractSettingsGitLog(extractLogMessageFromGitCommit("p4")) - if len(depotPath) == 0 and gitBranchExists("origin"): - settings = extractSettingsGitLog(extractLogMessageFromGitCommit("origin")) - depotPaths = settings['depot-paths'] + parent = 0 + while parent < 65535: + commit = "HEAD~%s" % parent + log = extractLogMessageFromGitCommit(commit) + settings = extractSettingsGitLog(log) + if not settings.has_key("depot-paths"): + parent = parent + 1 + continue + + depotPath = settings['depot-paths'][0] + + if len(self.origin) == 0: + names = read_pipe_lines("git name-rev '--refs=refs/remotes/p4/*' '%s'" % commit) + if len(names) > 0: + # strip away the beginning of 'HEAD~42 refs/remotes/p4/foo' + self.origin = names[0].strip()[len(commit) + 1:] + + break + + if self.verbose: + print "Origin branch is " + self.origin if len(depotPath) == 0: print "Internal error: cannot locate perforce depot path from existing branches" @@ -510,7 +536,7 @@ class P4Submit(Command): print "No changes in working directory to submit." return True patch = read_pipe("git diff -p --binary --diff-filter=ACMRTUXB HEAD") - self.diffFile = gitdir + "/p4-git-diff" + self.diffFile = self.gitdir + "/p4-git-diff" f = open(self.diffFile, "wb") f.write(patch) f.close(); @@ -520,12 +546,6 @@ class P4Submit(Command): if response == "y" or response == "yes": system("p4 sync ...") - if len(self.origin) == 0: - if gitBranchExists("p4"): - self.origin = "p4" - else: - self.origin = "origin" - if self.reset: self.firstTime = True @@ -535,7 +555,7 @@ class P4Submit(Command): self.logSubstitutions[tokens[0]] = tokens[1] self.check() - self.configFile = gitdir + "/p4-git-sync.cfg" + self.configFile = self.gitdir + "/p4-git-sync.cfg" self.config = shelve.open(self.configFile, writeback=True) if self.firstTime: @@ -581,9 +601,11 @@ class P4Sync(Command): optparse.make_option("--silent", dest="silent", action="store_true"), optparse.make_option("--detect-labels", dest="detectLabels", action="store_true"), optparse.make_option("--verbose", dest="verbose", action="store_true"), - optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false"), + optparse.make_option("--import-local", dest="importIntoRemotes", action="store_false", + help="Import into refs/heads/ , not refs/remotes"), optparse.make_option("--max-changes", dest="maxChanges"), - optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true') + optparse.make_option("--keep-path", dest="keepRepoPath", action='store_true', + help="Keep entire BRANCH/DIR/SUBDIR prefix during import") ] self.description = """Imports from Perforce into a git repository.\n example: @@ -612,9 +634,6 @@ class P4Sync(Command): if gitConfig("git-p4.syncFromOrigin") == "false": self.syncWithOrigin = False - def p4File(self, depotPath): - return read_pipe("p4 print -q \"%s\"" % depotPath) - def extractFilesFromCommit(self, commit): files = [] fnum = 0 @@ -676,6 +695,40 @@ class P4Sync(Command): return branches + ## Should move this out, doesn't use SELF. + def readP4Files(self, files): + files = [f for f in files + if f['action'] != 'delete'] + + if not files: + return + + filedata = p4CmdList('print %s' % ' '.join(['"%s#%s"' % (f['path'], + f['rev']) + for f in files])) + + j = 0; + contents = {} + while j < len(filedata): + stat = filedata[j] + j += 1 + text = '' + while j < len(filedata) and filedata[j]['code'] in ('text', + 'binary'): + text += filedata[j]['data'] + j += 1 + + + if not stat.has_key('depotFile'): + sys.stderr.write("p4 print fails with: %s\n" % repr(stat)) + continue + + contents[stat['depotFile']] = text + + for f in files: + assert not f.has_key('data') + f['data'] = contents[f['path']] + def commit(self, details, files, branch, branchPrefixes, parent = ""): epoch = details["time"] author = details["user"] @@ -683,8 +736,22 @@ class P4Sync(Command): if self.verbose: print "commit into %s" % branch + # start with reading files; if that fails, we should not + # create a commit. + new_files = [] + for f in files: + if [p for p in branchPrefixes if f['path'].startswith(p)]: + new_files.append (f) + else: + sys.stderr.write("Ignoring file outside of prefix: %s\n" % path) + files = new_files + self.readP4Files(files) + + + + self.gitStream.write("commit %s\n" % branch) - # gitStream.write("mark :%s\n" % details["change"]) +# gitStream.write("mark :%s\n" % details["change"]) self.committedChanges.add(int(details["change"])) committer = "" if author not in self.users: @@ -711,33 +778,24 @@ class P4Sync(Command): self.gitStream.write("from %s\n" % parent) for file in files: - path = file["path"] - - - if not [p for p in branchPrefixes if path.startswith(p)]: - continue - rev = file["rev"] - depotPath = path + "#" + rev - relPath = self.stripRepoPath(path, branchPrefixes) - action = file["action"] - if file["type"] == "apple": - print "\nfile %s is a strange apple file that forks. Ignoring!" % path + print "\nfile %s is a strange apple file that forks. Ignoring!" % file['path'] continue - if action == "delete": + relPath = self.stripRepoPath(file['path'], branchPrefixes) + if file["action"] == "delete": self.gitStream.write("D %s\n" % relPath) else: mode = 644 if file["type"].startswith("x"): mode = 755 - data = self.p4File(depotPath) + data = file['data'] if self.isWindows and file["type"].endswith("text"): data = data.replace("\r\n", "\n") - self.gitStream.write("M %s inline %s\n" % (mode, relPath)) + self.gitStream.write("M %d inline %s\n" % (mode, relPath)) self.gitStream.write("data %s\n" % len(data)) self.gitStream.write(data) self.gitStream.write("\n") @@ -789,6 +847,9 @@ class P4Sync(Command): print ("Tag %s does not match with change %s: file count is different." % (labelDetails["label"], change)) + def getUserCacheFilename(self): + return os.environ["HOME"] + "/.gitp4-usercache.txt" + def getUserMapFromPerforceServer(self): if self.userMapFromPerforceServer: return @@ -799,17 +860,19 @@ class P4Sync(Command): continue self.users[output["User"]] = output["FullName"] + " <" + output["Email"] + ">" - cache = open(gitdir + "/p4-usercache.txt", "wb") - for user in self.users.keys(): - cache.write("%s\t%s\n" % (user, self.users[user])) - cache.close(); + + s = '' + for (key, val) in self.users.items(): + s += "%s\t%s\n" % (key, val) + + open(self.getUserCacheFilename(), "wb").write(s) self.userMapFromPerforceServer = True def loadUserMapFromCache(self): self.users = {} self.userMapFromPerforceServer = False try: - cache = open(gitdir + "/p4-usercache.txt", "rb") + cache = open(self.getUserCacheFilename(), "rb") lines = cache.readlines() cache.close() for line in lines: @@ -844,11 +907,11 @@ class P4Sync(Command): if self.verbose: print "Label changes: %s" % self.labels.keys() - def getBranchMapping(self): - - ## FIXME - what's a P4 projectName ? - self.projectName = self.depotPath[self.depotPath.strip().rfind("/") + 1:] + def guessProjectName(self): + for p in self.depotPaths: + return p [p.strip().rfind("/") + 1:] + def getBranchMapping(self): for info in p4CmdList("branches"): details = p4Cmd("branch -o %s" % info["branch"]) viewIdx = 0 @@ -860,9 +923,10 @@ class P4Sync(Command): continue source = paths[0] destination = paths[1] - if source.startswith(self.depotPath) and destination.startswith(self.depotPath): - source = source[len(self.depotPath):-4] - destination = destination[len(self.depotPath):-4] + ## HACK + if source.startswith(self.depotPaths[0]) and destination.startswith(self.depotPaths[0]): + source = source[len(self.depotPaths[0]):-4] + destination = destination[len(self.depotPaths[0]):-4] if destination not in self.knownBranches: self.knownBranches[destination] = source if source not in self.knownBranches: @@ -878,24 +942,27 @@ class P4Sync(Command): cmdline += " --branches" for line in read_pipe_lines(cmdline): - lie = line.strip() - if self.importIntoRemotes and ((not line.startswith("p4/")) or line == "p4/HEAD\n"): + line = line.strip() + + ## only import to p4/ + if not line.startswith('p4/') or line == "p4/HEAD": continue + branch = line - if self.importIntoRemotes: - # strip off p4 - branch = re.sub ("^p4/", "", line) + # strip off p4 + branch = re.sub ("^p4/", "", line) self.p4BranchesInGit.append(branch) self.initialParents[self.refPrefix + branch] = parseRevision(line) def createOrUpdateBranchesFromOrigin(self): if not self.silent: - print "Creating/updating branch(es) in %s based on origin branch(es)" % self.refPrefix + print ("Creating/updating branch(es) in %s based on origin branch(es)" + % self.refPrefix) for line in read_pipe_lines("git rev-parse --symbolic --remotes"): line = line.strip() - if (not line.startswith("origin/")) or line.endswith("HEAD\n"): + if (not line.startswith("origin/")) or line.endswith("HEAD"): continue headName = line[len("origin/"):] @@ -913,7 +980,7 @@ class P4Sync(Command): print "creating %s" % remoteHead update = True else: - settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) + settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) if settings.has_key('change') > 0: if settings['depot-paths'] == original['depot-paths']: originP4Change = int(original['change']) @@ -958,7 +1025,7 @@ class P4Sync(Command): if self.importIntoRemotes: self.refPrefix = "refs/remotes/p4/" else: - self.refPrefix = "refs/heads/" + self.refPrefix = "refs/heads/p4/" if self.syncWithOrigin and self.hasOrigin: if not self.silent: @@ -974,8 +1041,9 @@ class P4Sync(Command): if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) - ### FIXME - if 1: + # TODO: should always look at previous commits, + # merge with previous imports, if possible. + if args == []: if self.hasOrigin: self.createOrUpdateBranchesFromOrigin() self.listExistingP4GitBranches() @@ -994,9 +1062,6 @@ class P4Sync(Command): settings = extractSettingsGitLog(logMsg) - if self.verbose: - print "path %s change %s" % (','.join(depotPaths), change) - self.readOptions(settings) if (settings.has_key('depot-paths') and settings.has_key ('change')): @@ -1009,18 +1074,20 @@ class P4Sync(Command): else: paths = [] for (prev, cur) in zip(self.previousDepotPaths, depotPaths): - for i in range(0, max(len(cur), len(prev))): + for i in range(0, min(len(cur), len(prev))): if cur[i] <> prev[i]: + i = i - 1 break - paths.append (cur[:i]) + paths.append (cur[:i + 1]) self.previousDepotPaths = paths if p4Change > 0: self.depotPaths = sorted(self.previousDepotPaths) self.changeRange = "@%s,#head" % p4Change - self.initialParent = parseRevision(self.branch) + if not self.detectBranches: + self.initialParent = parseRevision(self.branch) if not self.silent and not self.detectBranches: print "Performing incremental import into %s git branch" % self.branch @@ -1049,7 +1116,7 @@ class P4Sync(Command): self.changeRange = p[atIdx:] if self.changeRange == "@all": self.changeRange = "" - elif self.changeRange.find(",") == -1: + elif ',' not in self.changeRange: self.revision = self.changeRange self.changeRange = "" p = p[0:atIdx] @@ -1075,7 +1142,11 @@ class P4Sync(Command): self.getLabels(); if self.detectBranches: - self.getBranchMapping(); + ## FIXME - what's a P4 projectName ? + self.projectName = self.guessProjectName() + + if not self.hasOrigin: + self.getBranchMapping(); if self.verbose: print "p4-git branches: %s" % self.p4BranchesInGit print "initial parents: %s" % self.initialParents @@ -1095,7 +1166,7 @@ class P4Sync(Command): self.gitStream = importProcess.stdin self.gitError = importProcess.stderr - if len(self.revision) > 0: + if self.revision: print "Doing initial import of %s from revision %s" % (' '.join(self.depotPaths), self.revision) details = { "user" : "git perforce import user", "time" : int(time.time()) } @@ -1109,6 +1180,13 @@ class P4Sync(Command): + ' '.join(["%s...%s" % (p, self.revision) for p in self.depotPaths])): + + if info['code'] == 'error': + sys.stderr.write("p4 returned an error: %s\n" + % info['data']) + sys.exit(1) + + change = int(info["change"]) if change > newestRevision: newestRevision = change @@ -1118,7 +1196,7 @@ class P4Sync(Command): #fileCnt = fileCnt + 1 continue - for prop in [ "depotFile", "rev", "action", "type" ]: + for prop in ["depotFile", "rev", "action", "type" ]: details["%s%s" % (prop, fileCnt)] = info[prop] fileCnt = fileCnt + 1 @@ -1146,7 +1224,7 @@ class P4Sync(Command): changes.sort() else: if self.verbose: - print "Getting p4 changes for %s...%s" % (`self.depotPaths`, + print "Getting p4 changes for %s...%s" % (', '.join(self.depotPaths), self.changeRange) assert self.depotPaths output = read_pipe_lines("p4 changes " + ' '.join (["%s...%s" % (p, self.changeRange) @@ -1260,6 +1338,7 @@ class P4Rebase(Command): self.options = [ ] self.description = ("Fetches the latest revision from perforce and " + "rebases the current work (branch) against it") + self.verbose = False def run(self, args): sync = P4Sync() @@ -1282,9 +1361,16 @@ class P4Clone(P4Sync): self.cloneDestination = None self.needsGit = False - def run(self, args): - global gitdir + def defaultDestination(self, args): + ## TODO: use common prefix of args? + depotPath = args[0] + depotDir = re.sub("(@[^@]*)$", "", depotPath) + depotDir = re.sub("(#[^#]*)$", "", depotDir) + depotDir = re.sub(r"\.\.\.$,", "", depotDir) + depotDir = re.sub(r"/$", "", depotDir) + return os.path.split(depotDir)[1] + def run(self, args): if len(args) < 1: return False @@ -1293,24 +1379,23 @@ class P4Clone(P4Sync): sys.exit(1) depotPaths = args + + if not self.cloneDestination and len(depotPaths) > 1: + self.cloneDestination = depotPaths[-1] + depotPaths = depotPaths[:-1] + for p in depotPaths: if not p.startswith("//"): return False if not self.cloneDestination: - depotPath = args[0] - depotDir = re.sub("(@[^@]*)$", "", depotPath) - depotDir = re.sub("(#[^#]*)$", "", depotDir) - depotDir = re.sub(r"\.\.\.$,", "", depotDir) - depotDir = re.sub(r"/$", "", depotDir) + self.cloneDestination = self.defaultDestination(args) - self.cloneDestination = os.path.split(depotDir)[1] - - print "Importing from %s into %s" % (`depotPaths`, self.cloneDestination) + print "Importing from %s into %s" % (', '.join(depotPaths), self.cloneDestination) os.makedirs(self.cloneDestination) os.chdir(self.cloneDestination) system("git init") - gitdir = os.getcwd() + "/.git" + self.gitdir = os.getcwd() + "/.git" if not P4Sync.run(self, depotPaths): return False if self.branch != "master": @@ -1319,6 +1404,7 @@ class P4Clone(P4Sync): system("git checkout -f") else: print "Could not detect main branch. No checkout/master branch created." + return True class HelpFormatter(optparse.IndentedHelpFormatter): @@ -1340,12 +1426,12 @@ def printUsage(commands): print "" commands = { - "debug" : P4Debug(), - "submit" : P4Submit(), - "sync" : P4Sync(), - "rebase" : P4Rebase(), - "clone" : P4Clone(), - "rollback" : P4RollBack() + "debug" : P4Debug, + "submit" : P4Submit, + "sync" : P4Sync, + "rebase" : P4Rebase, + "clone" : P4Clone, + "rollback" : P4RollBack } @@ -1357,7 +1443,8 @@ def main(): cmd = "" cmdName = sys.argv[1] try: - cmd = commands[cmdName] + klass = commands[cmdName] + cmd = klass() except KeyError: print "unknown command %s" % cmdName print "" @@ -1365,7 +1452,7 @@ def main(): sys.exit(2) options = cmd.options - cmd.gitdir = gitdir + cmd.gitdir = os.environ.get("GIT_DIR", None) args = sys.argv[2:] @@ -1381,23 +1468,22 @@ def main(): global verbose verbose = cmd.verbose if cmd.needsGit: - gitdir = cmd.gitdir - if len(gitdir) == 0: - gitdir = ".git" - if not isValidGitDir(gitdir): - gitdir = read_pipe("git rev-parse --git-dir").strip() - if os.path.exists(gitdir): + if cmd.gitdir == None: + cmd.gitdir = os.path.abspath(".git") + if not isValidGitDir(cmd.gitdir): + cmd.gitdir = read_pipe("git rev-parse --git-dir").strip() + if os.path.exists(cmd.gitdir): cdup = read_pipe("git rev-parse --show-cdup").strip() if len(cdup) > 0: os.chdir(cdup); - if not isValidGitDir(gitdir): - if isValidGitDir(gitdir + "/.git"): - gitdir += "/.git" + if not isValidGitDir(cmd.gitdir): + if isValidGitDir(cmd.gitdir + "/.git"): + cmd.gitdir += "/.git" else: - die("fatal: cannot locate git repository at %s" % gitdir) + die("fatal: cannot locate git repository at %s" % cmd.gitdir) - os.environ["GIT_DIR"] = gitdir + os.environ["GIT_DIR"] = cmd.gitdir if not cmd.run(args): parser.print_help()