X-Git-Url: https://git.tokkee.org/?a=blobdiff_plain;f=contrib%2Ffast-import%2Fgit-p4;h=55778c577564dc2afde8cf36f3dbc1c994e4ded4;hb=9656153b87d47b34986f2d77eef9199c24cbf9f6;hp=54a05eb99c3eb747fcf26cfa3fc51a12f9055bd3;hpb=5c054a985a5b7e758f48353ac03582c9b2a9f919;p=git.git diff --git a/contrib/fast-import/git-p4 b/contrib/fast-import/git-p4 index 54a05eb99..55778c577 100755 --- a/contrib/fast-import/git-p4 +++ b/contrib/fast-import/git-p4 @@ -63,21 +63,34 @@ def system(cmd): if os.system(cmd) != 0: die("command failed: %s" % cmd) -def p4CmdList(cmd): +def p4CmdList(cmd, stdin=None, stdin_mode='w+b'): cmd = "p4 -G %s" % cmd if verbose: sys.stderr.write("Opening pipe: %s\n" % cmd) - pipe = os.popen(cmd, "rb") + + # Use a temporary file to avoid deadlocks without + # subprocess.communicate(), which would put another copy + # of stdout into memory. + stdin_file = None + if stdin is not None: + stdin_file = tempfile.TemporaryFile(prefix='p4-stdin', mode=stdin_mode) + stdin_file.write(stdin) + stdin_file.flush() + stdin_file.seek(0) + + p4 = subprocess.Popen(cmd, shell=True, + stdin=stdin_file, + stdout=subprocess.PIPE) result = [] try: while True: - entry = marshal.load(pipe) + entry = marshal.load(p4.stdout) result.append(entry) except EOFError: pass - exitCode = pipe.close() - if exitCode != None: + exitCode = p4.wait() + if exitCode != 0: entry = {} entry["p4ExitCode"] = exitCode result.append(entry) @@ -168,27 +181,105 @@ def gitBranchExists(branch): def gitConfig(key): return read_pipe("git config %s" % key, ignore_error=True).strip() +def p4BranchesInGit(branchesAreInRemotes = True): + branches = {} + + cmdline = "git rev-parse --symbolic " + if branchesAreInRemotes: + cmdline += " --remotes" + else: + cmdline += " --branches" + + for line in read_pipe_lines(cmdline): + line = line.strip() + + ## only import to p4/ + if not line.startswith('p4/') or line == "p4/HEAD": + continue + branch = line + + # strip off p4 + branch = re.sub ("^p4/", "", line) + + branches[branch] = parseRevision(line) + return branches + def findUpstreamBranchPoint(head = "HEAD"): + branches = p4BranchesInGit() + # map from depot-path to branch name + branchByDepotPath = {} + for branch in branches.keys(): + tip = branches[branch] + log = extractLogMessageFromGitCommit(tip) + settings = extractSettingsGitLog(log) + if settings.has_key("depot-paths"): + paths = ",".join(settings["depot-paths"]) + branchByDepotPath[paths] = "remotes/p4/" + branch + settings = None - branchPoint = "" parent = 0 while parent < 65535: commit = head + "~%s" % parent log = extractLogMessageFromGitCommit(commit) settings = extractSettingsGitLog(log) - if not settings.has_key("depot-paths"): - parent = parent + 1 + if settings.has_key("depot-paths"): + paths = ",".join(settings["depot-paths"]) + if branchByDepotPath.has_key(paths): + return [branchByDepotPath[paths], settings] + + parent = parent + 1 + + return ["", settings] + +def createOrUpdateBranchesFromOrigin(localRefPrefix = "refs/remotes/p4/", silent=True): + if not silent: + print ("Creating/updating branch(es) in %s based on origin branch(es)" + % localRefPrefix) + + originPrefix = "origin/p4/" + + for line in read_pipe_lines("git rev-parse --symbolic --remotes"): + line = line.strip() + if (not line.startswith(originPrefix)) or line.endswith("HEAD"): continue - names = read_pipe_lines("git name-rev \"--refs=refs/remotes/p4/*\" \"%s\"" % commit) - if len(names) <= 0: + headName = line[len(originPrefix):] + remoteHead = localRefPrefix + headName + originHead = line + + original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) + if (not original.has_key('depot-paths') + or not original.has_key('change')): continue - # strip away the beginning of 'HEAD~42 refs/remotes/p4/foo' - branchPoint = names[0].strip()[len(commit) + 1:] - break + update = False + if not gitBranchExists(remoteHead): + if verbose: + print "creating %s" % remoteHead + update = True + else: + settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) + if settings.has_key('change') > 0: + if settings['depot-paths'] == original['depot-paths']: + originP4Change = int(original['change']) + p4Change = int(settings['change']) + if originP4Change > p4Change: + print ("%s (%s) is newer than %s (%s). " + "Updating p4 branch from origin." + % (originHead, originP4Change, + remoteHead, p4Change)) + update = True + else: + print ("Ignoring: %s was imported from %s while " + "%s was imported from %s" + % (originHead, ','.join(original['depot-paths']), + remoteHead, ','.join(settings['depot-paths']))) + + if update: + system("git update-ref %s %s" % (remoteHead, originHead)) - return [branchPoint, settings] +def originP4BranchesExist(): + return gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") class Command: def __init__(self): @@ -349,6 +440,30 @@ class P4Submit(Command): return result + def prepareSubmitTemplate(self): + # remove lines in the Files section that show changes to files outside the depot path we're committing into + template = "" + inFilesSection = False + for line in read_pipe_lines("p4 change -o"): + if inFilesSection: + if line.startswith("\t"): + # path starts and ends with a tab + path = line[1:] + lastTab = path.rfind("\t") + if lastTab != -1: + path = path[:lastTab] + if not path.startswith(self.depotPath): + continue + else: + inFilesSection = False + else: + if line.startswith("Files:"): + inFilesSection = True + + template += line + + return template + def applyCommit(self, id): if self.directSubmit: print "Applying local change in working directory/index" @@ -426,7 +541,7 @@ class P4Submit(Command): logMessage = logMessage.replace("\n", "\r\n") logMessage = logMessage.strip() - template = read_pipe("p4 change -o") + template = self.prepareSubmitTemplate() if self.interactive: submitTemplate = self.prepareLogMessage(template, logMessage) @@ -517,24 +632,24 @@ class P4Submit(Command): return False [upstream, settings] = findUpstreamBranchPoint() - depotPath = settings['depot-paths'][0] + self.depotPath = settings['depot-paths'][0] if len(self.origin) == 0: self.origin = upstream if self.verbose: print "Origin branch is " + self.origin - if len(depotPath) == 0: + if len(self.depotPath) == 0: print "Internal error: cannot locate perforce depot path from existing branches" sys.exit(128) - self.clientPath = p4Where(depotPath) + self.clientPath = p4Where(self.depotPath) if len(self.clientPath) == 0: - print "Error: Cannot locate perforce checkout of %s in client view" % depotPath + print "Error: Cannot locate perforce checkout of %s in client view" % self.depotPath sys.exit(128) - print "Perforce checkout for depot path %s located at %s" % (depotPath, self.clientPath) + print "Perforce checkout for depot path %s located at %s" % (self.depotPath, self.clientPath) self.oldWorkingDirectory = os.getcwd() if self.directSubmit: @@ -712,23 +827,13 @@ class P4Sync(Command): if not files: return - # We cannot put all the files on the command line - # OS have limitations on the max lenght of arguments - # POSIX says it's 4096 bytes, default for Linux seems to be 130 K. - # and all OS from the table below seems to be higher than POSIX. - # See http://www.in-ulm.de/~mascheck/various/argmax/ - argmax = min(4000, os.sysconf('SC_ARG_MAX')) - chunk = '' - filedata = [] - for i in xrange(len(files)): - f = files[i] - chunk += '"%s#%s" ' % (f['path'], f['rev']) - if len(chunk) > argmax or i == len(files)-1: - data = p4CmdList('print %s' % chunk) - if "p4ExitCode" in data[0]: - die("Problems executing p4. Error: [%d]." % (data[0]['p4ExitCode'])); - filedata.extend(data) - chunk = '' + filedata = p4CmdList('-x - print', + stdin='\n'.join(['%s#%s' % (f['path'], f['rev']) + for f in files]), + stdin_mode='w+') + if "p4ExitCode" in filedata[0]: + die("Problems executing p4. Error: [%d]." + % (filedata[0]['p4ExitCode'])); j = 0; contents = {} @@ -808,16 +913,20 @@ class P4Sync(Command): if file["action"] == "delete": self.gitStream.write("D %s\n" % relPath) else: - mode = 644 - if file["type"].startswith("x"): - mode = 755 - data = file['data'] + mode = "644" + if file["type"].startswith("x"): + mode = "755" + elif file["type"] == "symlink": + mode = "120000" + # p4 print on a symlink contains "target\n", so strip it off + data = data[:-1] + if self.isWindows and file["type"].endswith("text"): data = data.replace("\r\n", "\n") - self.gitStream.write("M %d inline %s\n" % (mode, relPath)) + self.gitStream.write("M %s inline %s\n" % (mode, relPath)) self.gitStream.write("data %s\n" % len(data)) self.gitStream.write(data) self.gitStream.write("\n") @@ -870,7 +979,8 @@ class P4Sync(Command): % (labelDetails["label"], change)) def getUserCacheFilename(self): - return os.environ["HOME"] + "/.gitp4-usercache.txt" + home = os.environ.get("HOME", os.environ.get("USERPROFILE")) + return home + "/.gitp4-usercache.txt" def getUserMapFromPerforceServer(self): if self.userMapFromPerforceServer: @@ -975,74 +1085,11 @@ class P4Sync(Command): self.knownBranches[branch] = branch def listExistingP4GitBranches(self): - self.p4BranchesInGit = [] - - cmdline = "git rev-parse --symbolic " - if self.importIntoRemotes: - cmdline += " --remotes" - else: - cmdline += " --branches" - - for line in read_pipe_lines(cmdline): - line = line.strip() - - ## only import to p4/ - if not line.startswith('p4/') or line == "p4/HEAD": - continue - branch = line - - # strip off p4 - branch = re.sub ("^p4/", "", line) - - self.p4BranchesInGit.append(branch) - self.initialParents[self.refPrefix + branch] = parseRevision(line) - - def createOrUpdateBranchesFromOrigin(self): - if not self.silent: - print ("Creating/updating branch(es) in %s based on origin branch(es)" - % self.refPrefix) - - originPrefix = "origin/p4/" - - for line in read_pipe_lines("git rev-parse --symbolic --remotes"): - line = line.strip() - if (not line.startswith(originPrefix)) or line.endswith("HEAD"): - continue - - headName = line[len(originPrefix):] - remoteHead = self.refPrefix + headName - originHead = line - - original = extractSettingsGitLog(extractLogMessageFromGitCommit(originHead)) - if (not original.has_key('depot-paths') - or not original.has_key('change')): - continue - - update = False - if not gitBranchExists(remoteHead): - if self.verbose: - print "creating %s" % remoteHead - update = True - else: - settings = extractSettingsGitLog(extractLogMessageFromGitCommit(remoteHead)) - if settings.has_key('change') > 0: - if settings['depot-paths'] == original['depot-paths']: - originP4Change = int(original['change']) - p4Change = int(settings['change']) - if originP4Change > p4Change: - print ("%s (%s) is newer than %s (%s). " - "Updating p4 branch from origin." - % (originHead, originP4Change, - remoteHead, p4Change)) - update = True - else: - print ("Ignoring: %s was imported from %s while " - "%s was imported from %s" - % (originHead, ','.join(original['depot-paths']), - remoteHead, ','.join(settings['depot-paths']))) - - if update: - system("git update-ref %s %s" % (remoteHead, originHead)) + # branches holds mapping from name to commit + branches = p4BranchesInGit(self.importIntoRemotes) + self.p4BranchesInGit = branches.keys() + for branch in branches.keys(): + self.initialParents[self.refPrefix + branch] = branches[branch] def updateOptionDict(self, d): option_keys = {} @@ -1064,7 +1111,7 @@ class P4Sync(Command): # map from branch depot path to parent branch self.knownBranches = {} self.initialParents = {} - self.hasOrigin = gitBranchExists("origin") or gitBranchExists("origin/p4") or gitBranchExists("origin/p4/master") + self.hasOrigin = originP4BranchesExist() if not self.syncWithOrigin: self.hasOrigin = False @@ -1084,14 +1131,14 @@ class P4Sync(Command): system("git update-ref %s refs/heads/p4" % self.branch) system("git branch -D p4"); # create it /after/ importing, when master exists - if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes: + if not gitBranchExists(self.refPrefix + "HEAD") and self.importIntoRemotes and gitBranchExists(self.branch): system("git symbolic-ref %sHEAD %s" % (self.refPrefix, self.branch)) # TODO: should always look at previous commits, # merge with previous imports, if possible. if args == []: if self.hasOrigin: - self.createOrUpdateBranchesFromOrigin() + createOrUpdateBranchesFromOrigin(self.refPrefix, self.silent) self.listExistingP4GitBranches() if len(self.p4BranchesInGit) > 1: @@ -1165,11 +1212,11 @@ class P4Sync(Command): elif ',' not in self.changeRange: self.revision = self.changeRange self.changeRange = "" - p = p[0:atIdx] + p = p[:atIdx] elif p.find("#") != -1: hashIdx = p.index("#") self.revision = p[hashIdx:] - p = p[0:hashIdx] + p = p[:hashIdx] elif self.previousDepotPaths == []: self.revision = "#head" @@ -1278,12 +1325,12 @@ class P4Sync(Command): for line in output: changeNum = line.split(" ")[1] - changes.append(changeNum) + changes.append(int(changeNum)) - changes.reverse() + changes.sort() if len(self.maxChanges) > 0: - changes = changes[0:min(int(self.maxChanges), len(changes))] + changes = changes[:min(int(self.maxChanges), len(changes))] if len(changes) == 0: if not self.silent: @@ -1474,6 +1521,9 @@ class P4Branches(Command): self.verbose = False def run(self, args): + if originP4BranchesExist(): + createOrUpdateBranchesFromOrigin() + cmdline = "git rev-parse --symbolic " cmdline += " --remotes"