8a4cca3b3e73de2edd5561aaf82efd90117be0eb
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.16 2001-12-12 03:23:14 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal
27 from roundup import hyperdb, date, password
29 #
30 # Now the database
31 #
32 class Database(hyperdb.Database):
33 """A database for storing records containing flexible data types.
35 Transaction stuff TODO:
36 . check the timestamp of the class file and nuke the cache if it's
37 modified. Do some sort of conflict checking on the dirty stuff.
38 . perhaps detect write collisions (related to above)?
40 """
41 def __init__(self, storagelocator, journaltag=None):
42 """Open a hyperdatabase given a specifier to some storage.
44 The meaning of 'storagelocator' depends on the particular
45 implementation of the hyperdatabase. It could be a file name,
46 a directory path, a socket descriptor for a connection to a
47 database over the network, etc.
49 The 'journaltag' is a token that will be attached to the journal
50 entries for any edits done on the database. If 'journaltag' is
51 None, the database is opened in read-only mode: the Class.create(),
52 Class.set(), and Class.retire() methods are disabled.
53 """
54 self.dir, self.journaltag = storagelocator, journaltag
55 self.classes = {}
56 self.cache = {} # cache of nodes loaded or created
57 self.dirtynodes = {} # keep track of the dirty nodes by class
58 self.newnodes = {} # keep track of the new nodes by class
59 self.transactions = []
61 #
62 # Classes
63 #
64 def __getattr__(self, classname):
65 """A convenient way of calling self.getclass(classname)."""
66 return self.classes[classname]
68 def addclass(self, cl):
69 cn = cl.classname
70 if self.classes.has_key(cn):
71 raise ValueError, cn
72 self.classes[cn] = cl
74 def getclasses(self):
75 """Return a list of the names of all existing classes."""
76 l = self.classes.keys()
77 l.sort()
78 return l
80 def getclass(self, classname):
81 """Get the Class object representing a particular class.
83 If 'classname' is not a valid class name, a KeyError is raised.
84 """
85 return self.classes[classname]
87 #
88 # Class DBs
89 #
90 def clear(self):
91 '''Delete all database contents
92 '''
93 for cn in self.classes.keys():
94 for type in 'nodes', 'journals':
95 path = os.path.join(self.dir, 'journals.%s'%cn)
96 if os.path.exists(path):
97 os.remove(path)
98 elif os.path.exists(path+'.db'): # dbm appends .db
99 os.remove(path+'.db')
101 def getclassdb(self, classname, mode='r'):
102 ''' grab a connection to the class db that will be used for
103 multiple actions
104 '''
105 return self._opendb('nodes.%s'%classname, mode)
107 def _opendb(self, name, mode):
108 '''Low-level database opener that gets around anydbm/dbm
109 eccentricities.
110 '''
111 # determine which DB wrote the class file
112 db_type = ''
113 path = os.path.join(os.getcwd(), self.dir, name)
114 if os.path.exists(path):
115 db_type = whichdb.whichdb(path)
116 if not db_type:
117 raise hyperdb.DatabaseError, "Couldn't identify database type"
118 elif os.path.exists(path+'.db'):
119 # if the path ends in '.db', it's a dbm database, whether
120 # anydbm says it's dbhash or not!
121 db_type = 'dbm'
123 # new database? let anydbm pick the best dbm
124 if not db_type:
125 return anydbm.open(path, 'n')
127 # open the database with the correct module
128 try:
129 dbm = __import__(db_type)
130 except ImportError:
131 raise hyperdb.DatabaseError, \
132 "Couldn't open database - the required module '%s'"\
133 "is not available"%db_type
134 return dbm.open(path, mode)
136 #
137 # Nodes
138 #
139 def addnode(self, classname, nodeid, node):
140 ''' add the specified node to its class's db
141 '''
142 self.newnodes.setdefault(classname, {})[nodeid] = 1
143 self.cache.setdefault(classname, {})[nodeid] = node
144 self.savenode(classname, nodeid, node)
146 def setnode(self, classname, nodeid, node):
147 ''' change the specified node
148 '''
149 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
150 # can't set without having already loaded the node
151 self.cache[classname][nodeid] = node
152 self.savenode(classname, nodeid, node)
154 def savenode(self, classname, nodeid, node):
155 ''' perform the saving of data specified by the set/addnode
156 '''
157 self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
159 def getnode(self, classname, nodeid, cldb=None):
160 ''' add the specified node to its class's db
161 '''
162 # try the cache
163 cache = self.cache.setdefault(classname, {})
164 if cache.has_key(nodeid):
165 return cache[nodeid]
167 # get from the database and save in the cache
168 db = cldb or self.getclassdb(classname)
169 if not db.has_key(nodeid):
170 raise IndexError, nodeid
171 res = marshal.loads(db[nodeid])
172 cache[nodeid] = res
173 return res
175 def hasnode(self, classname, nodeid, cldb=None):
176 ''' add the specified node to its class's db
177 '''
178 # try the cache
179 cache = self.cache.setdefault(classname, {})
180 if cache.has_key(nodeid):
181 return 1
183 # not in the cache - check the database
184 db = cldb or self.getclassdb(classname)
185 res = db.has_key(nodeid)
186 return res
188 def countnodes(self, classname, cldb=None):
189 # include the new nodes not saved to the DB yet
190 count = len(self.newnodes.get(classname, {}))
192 # and count those in the DB
193 db = cldb or self.getclassdb(classname)
194 count = count + len(db.keys())
195 return count
197 def getnodeids(self, classname, cldb=None):
198 # start off with the new nodes
199 res = self.newnodes.get(classname, {}).keys()
201 db = cldb or self.getclassdb(classname)
202 res = res + db.keys()
203 return res
205 #
206 # Journal
207 #
208 def addjournal(self, classname, nodeid, action, params):
209 ''' Journal the Action
210 'action' may be:
212 'create' or 'set' -- 'params' is a dictionary of property values
213 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
214 'retire' -- 'params' is None
215 '''
216 self.transactions.append((self._doSaveJournal, (classname, nodeid,
217 action, params)))
219 def getjournal(self, classname, nodeid):
220 ''' get the journal for id
221 '''
222 # attempt to open the journal - in some rare cases, the journal may
223 # not exist
224 try:
225 db = self._opendb('journals.%s'%classname, 'r')
226 except anydbm.error, error:
227 if str(error) == "need 'c' or 'n' flag to open new db": return []
228 elif error.args[0] != 2: raise
229 return []
230 journal = marshal.loads(db[nodeid])
231 res = []
232 for entry in journal:
233 (nodeid, date_stamp, self.journaltag, action, params) = entry
234 date_obj = date.Date(date_stamp)
235 res.append((nodeid, date_obj, self.journaltag, action, params))
236 return res
239 #
240 # Basic transaction support
241 #
242 def commit(self):
243 ''' Commit the current transactions.
244 '''
245 # lock the DB
246 for method, args in self.transactions:
247 # TODO: optimise this, duh!
248 method(*args)
249 # unlock the DB
251 # all transactions committed, back to normal
252 self.cache = {}
253 self.dirtynodes = {}
254 self.newnodes = {}
255 self.transactions = []
257 def _doSaveNode(self, classname, nodeid, node):
258 db = self.getclassdb(classname, 'c')
259 # now save the marshalled data
260 db[nodeid] = marshal.dumps(node)
261 db.close()
263 def _doSaveJournal(self, classname, nodeid, action, params):
264 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
265 params)
266 db = anydbm.open(os.path.join(self.dir, 'journals.%s'%classname), 'c')
267 if db.has_key(nodeid):
268 s = db[nodeid]
269 l = marshal.loads(db[nodeid])
270 l.append(entry)
271 else:
272 l = [entry]
273 db[nodeid] = marshal.dumps(l)
274 db.close()
276 def rollback(self):
277 ''' Reverse all actions from the current transaction.
278 '''
279 self.cache = {}
280 self.dirtynodes = {}
281 self.newnodes = {}
282 self.transactions = []
284 #
285 #$Log: not supported by cvs2svn $
286 #Revision 1.15 2001/12/12 02:30:51 richard
287 #I fixed the problems with people whose anydbm was using the dbm module at the
288 #backend. It turns out the dbm module modifies the file name to append ".db"
289 #and my check to determine if we're opening an existing or new db just
290 #tested os.path.exists() on the filename. Well, no longer! We now perform a
291 #much better check _and_ cope with the anydbm implementation module changing
292 #too!
293 #I also fixed the backends __init__ so only ImportError is squashed.
294 #
295 #Revision 1.14 2001/12/10 22:20:01 richard
296 #Enabled transaction support in the bsddb backend. It uses the anydbm code
297 #where possible, only replacing methods where the db is opened (it uses the
298 #btree opener specifically.)
299 #Also cleaned up some change note generation.
300 #Made the backends package work with pydoc too.
301 #
302 #Revision 1.13 2001/12/02 05:06:16 richard
303 #. We now use weakrefs in the Classes to keep the database reference, so
304 # the close() method on the database is no longer needed.
305 # I bumped the minimum python requirement up to 2.1 accordingly.
306 #. #487480 ] roundup-server
307 #. #487476 ] INSTALL.txt
308 #
309 #I also cleaned up the change message / post-edit stuff in the cgi client.
310 #There's now a clearly marked "TODO: append the change note" where I believe
311 #the change note should be added there. The "changes" list will obviously
312 #have to be modified to be a dict of the changes, or somesuch.
313 #
314 #More testing needed.
315 #
316 #Revision 1.12 2001/12/01 07:17:50 richard
317 #. We now have basic transaction support! Information is only written to
318 # the database when the commit() method is called. Only the anydbm
319 # backend is modified in this way - neither of the bsddb backends have been.
320 # The mail, admin and cgi interfaces all use commit (except the admin tool
321 # doesn't have a commit command, so interactive users can't commit...)
322 #. Fixed login/registration forwarding the user to the right page (or not,
323 # on a failure)
324 #
325 #Revision 1.11 2001/11/21 02:34:18 richard
326 #Added a target version field to the extended issue schema
327 #
328 #Revision 1.10 2001/10/09 23:58:10 richard
329 #Moved the data stringification up into the hyperdb.Class class' get, set
330 #and create methods. This means that the data is also stringified for the
331 #journal call, and removes duplication of code from the backends. The
332 #backend code now only sees strings.
333 #
334 #Revision 1.9 2001/10/09 07:25:59 richard
335 #Added the Password property type. See "pydoc roundup.password" for
336 #implementation details. Have updated some of the documentation too.
337 #
338 #Revision 1.8 2001/09/29 13:27:00 richard
339 #CGI interfaces now spit up a top-level index of all the instances they can
340 #serve.
341 #
342 #Revision 1.7 2001/08/12 06:32:36 richard
343 #using isinstance(blah, Foo) now instead of isFooType
344 #
345 #Revision 1.6 2001/08/07 00:24:42 richard
346 #stupid typo
347 #
348 #Revision 1.5 2001/08/07 00:15:51 richard
349 #Added the copyright/license notice to (nearly) all files at request of
350 #Bizar Software.
351 #
352 #Revision 1.4 2001/07/30 01:41:36 richard
353 #Makes schema changes mucho easier.
354 #
355 #Revision 1.3 2001/07/25 01:23:07 richard
356 #Added the Roundup spec to the new documentation directory.
357 #
358 #Revision 1.2 2001/07/23 08:20:44 richard
359 #Moved over to using marshal in the bsddb and anydbm backends.
360 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
361 # retired - mod hyperdb.Class.list() so it lists retired nodes)
362 #
363 #