From 7cd224cf545e2c71aef8116f4e3bdd97fdfa3d10 Mon Sep 17 00:00:00 2001 From: richard Date: Mon, 19 Aug 2002 02:53:27 +0000 Subject: [PATCH] full database export and import is done git-svn-id: http://svn.roundup-tracker.org/svnroot/roundup/trunk@968 57a73879-2fb5-44c3-a270-3262357dd7e2 --- roundup/admin.py | 153 ++++++++++++++------------------ roundup/backends/back_anydbm.py | 115 +++++++++++++++++++++++- 2 files changed, 177 insertions(+), 91 deletions(-) diff --git a/roundup/admin.py b/roundup/admin.py index 48b96d6..08fc092 100644 --- a/roundup/admin.py +++ b/roundup/admin.py @@ -16,7 +16,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -# $Id: admin.py,v 1.22 2002-08-16 04:26:42 richard Exp $ +# $Id: admin.py,v 1.23 2002-08-19 02:53:27 richard Exp $ import sys, os, getpass, getopt, re, UserDict, shlex, shutil try: @@ -808,13 +808,19 @@ Command help: return 0 def do_export(self, args): - '''Usage: export [class[,class]] destination_dir + '''Usage: export [class[,class]] export_dir Export the database to tab-separated-value files. This action exports the current data from the database into tab-separated-value files that are placed in the nominated destination directory. The journals are not exported. ''' + # we need the CSV module + if csv is None: + raise UsageError, \ + _('Sorry, you need the csv module to use this function.\n' + 'Get it from: http://www.object-craft.com.au/projects/csv/') + # grab the directory to export to if len(args) < 1: raise UsageError, _('Not enough arguments supplied') @@ -827,56 +833,38 @@ Command help: classes = self.db.classes.keys() # use the csv parser if we can - it's faster - if csv is not None: - p = csv.parser(field_sep=':') + p = csv.parser(field_sep=':') # do all the classes specified for classname in classes: cl = self.get_class(classname) f = open(os.path.join(dir, classname+'.csv'), 'w') - f.write(':'.join(cl.properties.keys()) + '\n') + properties = cl.getprops() + propnames = properties.keys() + propnames.sort() + print >> f, p.join(propnames) # all nodes for this class - properties = cl.getprops() for nodeid in cl.list(): - l = [] - for prop, proptype in properties: - value = cl.get(nodeid, prop) - # convert data where needed - if isinstance(proptype, hyperdb.Date): - value = value.get_tuple() - elif isinstance(proptype, hyperdb.Interval): - value = value.get_tuple() - elif isinstance(proptype, hyperdb.Password): - value = str(value) - l.append(repr(value)) - - # now write - if csv is not None: - f.write(p.join(l) + '\n') - else: - # escape the individual entries to they're valid CSV - m = [] - for entry in l: - if '"' in entry: - entry = '""'.join(entry.split('"')) - if ':' in entry: - entry = '"%s"'%entry - m.append(entry) - f.write(':'.join(m) + '\n') + print >>f, p.join(cl.export_list(propnames, nodeid)) return 0 def do_import(self, args): - '''Usage: import class file - Import the contents of the tab-separated-value file. - - The file must define the same properties as the class (including having - a "header" line with those property names.) The new nodes are added to - the existing database - if you want to create a new database using the - imported data, then create a new database (or, tediously, retire all - the old data.) + '''Usage: import import_dir + Import a database from the directory containing CSV files, one per + class to import. + + The files must define the same properties as the class (including having + a "header" line with those property names.) + + The imported nodes will have the same nodeid as defined in the + import file, thus replacing any existing content. + + XXX The new nodes are added to the existing database - if you want to + XXX create a new database using the imported data, then create a new + XXX database (or, tediously, retire all the old data.) ''' - if len(args) < 2: + if len(args) < 1: raise UsageError, _('Not enough arguments supplied') if csv is None: raise UsageError, \ @@ -885,56 +873,44 @@ Command help: from roundup import hyperdb - # ensure that the properties and the CSV file headings match - classname = args[0] - cl = self.get_class(classname) - f = open(args[1]) - p = csv.parser(field_sep=':') - file_props = p.parse(f.readline()) - props = cl.properties.keys() - m = file_props[:] - m.sort() - props.sort() - if m != props: - raise UsageError, _('Import file doesn\'t define the same ' - 'properties as "%(arg0)s".')%{'arg0': args[0]} - - # loop through the file and create a node for each entry - n = range(len(props)) - while 1: - line = f.readline() - if not line: break + for file in os.listdir(args[0]): + f = open(os.path.join(args[0], file)) - # parse lines until we get a complete entry + # get the classname + classname = os.path.splitext(file)[0] + + # ensure that the properties and the CSV file headings match + cl = self.get_class(classname) + p = csv.parser(field_sep=':') + file_props = p.parse(f.readline()) + properties = cl.getprops() + propnames = properties.keys() + propnames.sort() + m = file_props[:] + m.sort() + if m != propnames: + raise UsageError, _('Import file doesn\'t define the same ' + 'properties as "%(arg0)s".')%{'arg0': args[0]} + + # loop through the file and create a node for each entry + maxid = 1 while 1: - l = p.parse(line) - if l: break line = f.readline() - if not line: - raise ValueError, "Unexpected EOF during CSV parse" - - # make the new node's property map - d = {} - for i in n: - # Use eval to reverse the repr() used to output the CSV - value = eval(l[i]) - # Figure the property for this column - key = file_props[i] - proptype = cl.properties[key] - # Convert for property type - if isinstance(proptype, hyperdb.Date): - value = date.Date(value) - elif isinstance(proptype, hyperdb.Interval): - value = date.Interval(value) - elif isinstance(proptype, hyperdb.Password): - pwd = password.Password() - pwd.unpack(value) - value = pwd - if value is not None: - d[key] = value - - # and create the new node - apply(cl.create, (), d) + if not line: break + + # parse lines until we get a complete entry + while 1: + l = p.parse(line) + if l: break + line = f.readline() + if not line: + raise ValueError, "Unexpected EOF during CSV parse" + + # do the import and figure the current highest nodeid + maxid = max(maxid, int(cl.import_list(propnames, l))) + + print 'setting', classname, maxid + self.db.setid(classname, str(maxid)) return 0 def do_pack(self, args): @@ -1170,6 +1146,9 @@ if __name__ == '__main__': # # $Log: not supported by cvs2svn $ +# Revision 1.22 2002/08/16 04:26:42 richard +# moving towards full database export +# # Revision 1.21 2002/08/01 01:07:37 richard # include info about new user roles # diff --git a/roundup/backends/back_anydbm.py b/roundup/backends/back_anydbm.py index 6127105..27d8ae7 100644 --- a/roundup/backends/back_anydbm.py +++ b/roundup/backends/back_anydbm.py @@ -15,7 +15,7 @@ # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. # -#$Id: back_anydbm.py,v 1.60 2002-08-19 00:23:19 richard Exp $ +#$Id: back_anydbm.py,v 1.61 2002-08-19 02:53:27 richard Exp $ ''' This module defines a backend that saves the hyperdatabase in a database chosen by anydbm. It is guaranteed to always be available in python @@ -215,6 +215,16 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): release_lock(lock) return newid + def setid(self, classname, setid): + ''' Set the id counter: used during import of database + ''' + # open the ids DB - create if if doesn't exist + lock = self.lockdb('_ids') + db = self.opendb('_ids', 'c') + db[classname] = str(setid) + db.close() + release_lock(lock) + # # Nodes # @@ -593,13 +603,27 @@ class Database(FileStorage, hyperdb.Database, roundupdb.Database): return self.databases[db_name] def doSaveJournal(self, classname, nodeid, action, params): - # serialise first + # handle supply of the special journalling parameters (usually + # supplied on importing an existing database) + if params.has_key('creator'): + journaltag = self.user.get(params['creator'], 'username') + del params['creator'] + else: + journaltag = self.journaltag + if params.has_key('created'): + journaldate = params['created'].get_tuple() + del params['created'] + else: + journaldate = date.Date().get_tuple() + if params.has_key('activity'): + del params['activity'] + + # serialise the parameters now if action in ('set', 'create'): params = self.serialise(classname, params) # create the journal entry - entry = (nodeid, date.Date().get_tuple(), self.journaltag, action, - params) + entry = (nodeid, journaldate, journaltag, action, params) if __debug__: print >>hyperdb.DEBUG, 'doSaveJournal', entry @@ -845,6 +869,67 @@ class Class(hyperdb.Class): return newid + def export_list(self, propnames, nodeid): + ''' Export a node - generate a list of CSV-able data in the order + specified by propnames for the given node. + ''' + properties = self.getprops() + l = [] + for prop in propnames: + proptype = properties[prop] + value = self.get(nodeid, prop) + # "marshal" data where needed + if isinstance(proptype, hyperdb.Date): + value = value.get_tuple() + elif isinstance(proptype, hyperdb.Interval): + value = value.get_tuple() + elif isinstance(proptype, hyperdb.Password): + value = str(value) + l.append(repr(value)) + return l + + def import_list(self, propnames, proplist): + ''' Import a node - all information including "id" is present and + should not be sanity checked. Triggers are not triggered. The + journal should be initialised using the "creator" and "created" + information. + + Return the nodeid of the node imported. + ''' + if self.db.journaltag is None: + raise DatabaseError, 'Database open read-only' + properties = self.getprops() + + # make the new node's property map + d = {} + for i in range(len(propnames)): + # Use eval to reverse the repr() used to output the CSV + value = eval(proplist[i]) + + # Figure the property for this column + propname = propnames[i] + prop = properties[propname] + + # "unmarshal" where necessary + if propname == 'id': + newid = value + continue + elif isinstance(prop, hyperdb.Date): + value = date.Date(value) + elif isinstance(prop, hyperdb.Interval): + value = date.Interval(value) + elif isinstance(prop, hyperdb.Password): + pwd = password.Password() + pwd.unpack(value) + value = pwd + if value is not None: + d[propname] = value + + # add + self.db.addnode(self.classname, newid, d) + self.db.addjournal(self.classname, newid, 'create', d) + return newid + def get(self, nodeid, propname, default=_marker, cache=1): """Get the value of a property on an existing node of this class. @@ -1730,6 +1815,25 @@ class FileClass(Class): self.db.storefile(self.classname, newid, None, content) return newid + def import_list(self, propnames, proplist): + ''' Trap the "content" property... + ''' + # dupe this list so we don't affect others + propnames = propnames[:] + + # extract the "content" property from the proplist + i = propnames.index('content') + content = proplist[i] + del propnames[i] + del proplist[i] + + # do the normal import + newid = Class.import_list(self, propnames, proplist) + + # save off the "content" file + self.db.storefile(self.classname, newid, None, content) + return newid + def get(self, nodeid, propname, default=_marker, cache=1): ''' trap the content propname and get it from the file ''' @@ -1803,6 +1907,9 @@ class IssueClass(Class, roundupdb.IssueClass): # #$Log: not supported by cvs2svn $ +#Revision 1.60 2002/08/19 00:23:19 richard +#handle "unset" initial Link values (!) +# #Revision 1.59 2002/08/16 04:28:13 richard #added is_retired query to Class # -- 2.30.2