cca1c86c88e703c0c6ea33cc1e251c54bdd3a828
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.25 2002-01-22 05:06:08 rochecompaan Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal
27 from roundup import hyperdb, date, password
29 DEBUG=os.environ.get('HYPERDBDEBUG', '')
31 #
32 # Now the database
33 #
34 class Database(hyperdb.Database):
35 """A database for storing records containing flexible data types.
37 Transaction stuff TODO:
38 . check the timestamp of the class file and nuke the cache if it's
39 modified. Do some sort of conflict checking on the dirty stuff.
40 . perhaps detect write collisions (related to above)?
42 """
43 def __init__(self, config, journaltag=None):
44 """Open a hyperdatabase given a specifier to some storage.
46 The 'storagelocator' is obtained from config.DATABASE.
47 The meaning of 'storagelocator' depends on the particular
48 implementation of the hyperdatabase. It could be a file name,
49 a directory path, a socket descriptor for a connection to a
50 database over the network, etc.
52 The 'journaltag' is a token that will be attached to the journal
53 entries for any edits done on the database. If 'journaltag' is
54 None, the database is opened in read-only mode: the Class.create(),
55 Class.set(), and Class.retire() methods are disabled.
56 """
57 self.config, self.journaltag = config, journaltag
58 self.dir = config.DATABASE
59 self.classes = {}
60 self.cache = {} # cache of nodes loaded or created
61 self.dirtynodes = {} # keep track of the dirty nodes by class
62 self.newnodes = {} # keep track of the new nodes by class
63 self.transactions = []
65 def __repr__(self):
66 return '<back_anydbm instance at %x>'%id(self)
68 #
69 # Classes
70 #
71 def __getattr__(self, classname):
72 """A convenient way of calling self.getclass(classname)."""
73 if self.classes.has_key(classname):
74 if DEBUG:
75 print '__getattr__', (self, classname)
76 return self.classes[classname]
77 raise AttributeError, classname
79 def addclass(self, cl):
80 if DEBUG:
81 print 'addclass', (self, cl)
82 cn = cl.classname
83 if self.classes.has_key(cn):
84 raise ValueError, cn
85 self.classes[cn] = cl
87 def getclasses(self):
88 """Return a list of the names of all existing classes."""
89 if DEBUG:
90 print 'getclasses', (self,)
91 l = self.classes.keys()
92 l.sort()
93 return l
95 def getclass(self, classname):
96 """Get the Class object representing a particular class.
98 If 'classname' is not a valid class name, a KeyError is raised.
99 """
100 if DEBUG:
101 print 'getclass', (self, classname)
102 return self.classes[classname]
104 #
105 # Class DBs
106 #
107 def clear(self):
108 '''Delete all database contents
109 '''
110 if DEBUG:
111 print 'clear', (self,)
112 for cn in self.classes.keys():
113 for type in 'nodes', 'journals':
114 path = os.path.join(self.dir, 'journals.%s'%cn)
115 if os.path.exists(path):
116 os.remove(path)
117 elif os.path.exists(path+'.db'): # dbm appends .db
118 os.remove(path+'.db')
120 def getclassdb(self, classname, mode='r'):
121 ''' grab a connection to the class db that will be used for
122 multiple actions
123 '''
124 if DEBUG:
125 print 'getclassdb', (self, classname, mode)
126 return self._opendb('nodes.%s'%classname, mode)
128 def _opendb(self, name, mode):
129 '''Low-level database opener that gets around anydbm/dbm
130 eccentricities.
131 '''
132 if DEBUG:
133 print '_opendb', (self, name, mode)
134 # determine which DB wrote the class file
135 db_type = ''
136 path = os.path.join(os.getcwd(), self.dir, name)
137 if os.path.exists(path):
138 db_type = whichdb.whichdb(path)
139 if not db_type:
140 raise hyperdb.DatabaseError, "Couldn't identify database type"
141 elif os.path.exists(path+'.db'):
142 # if the path ends in '.db', it's a dbm database, whether
143 # anydbm says it's dbhash or not!
144 db_type = 'dbm'
146 # new database? let anydbm pick the best dbm
147 if not db_type:
148 if DEBUG:
149 print "_opendb anydbm.open(%r, 'n')"%path
150 return anydbm.open(path, 'n')
152 # open the database with the correct module
153 try:
154 dbm = __import__(db_type)
155 except ImportError:
156 raise hyperdb.DatabaseError, \
157 "Couldn't open database - the required module '%s'"\
158 "is not available"%db_type
159 if DEBUG:
160 print "_opendb %r.open(%r, %r)"%(db_type, path, mode)
161 return dbm.open(path, mode)
163 #
164 # Nodes
165 #
166 def addnode(self, classname, nodeid, node):
167 ''' add the specified node to its class's db
168 '''
169 if DEBUG:
170 print 'addnode', (self, classname, nodeid, node)
171 self.newnodes.setdefault(classname, {})[nodeid] = 1
172 self.cache.setdefault(classname, {})[nodeid] = node
173 self.savenode(classname, nodeid, node)
175 def setnode(self, classname, nodeid, node):
176 ''' change the specified node
177 '''
178 if DEBUG:
179 print 'setnode', (self, classname, nodeid, node)
180 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
181 # can't set without having already loaded the node
182 self.cache[classname][nodeid] = node
183 self.savenode(classname, nodeid, node)
185 def savenode(self, classname, nodeid, node):
186 ''' perform the saving of data specified by the set/addnode
187 '''
188 if DEBUG:
189 print 'savenode', (self, classname, nodeid, node)
190 self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
192 def getnode(self, classname, nodeid, db=None, cache=1):
193 ''' get a node from the database
194 '''
195 if DEBUG:
196 print 'getnode', (self, classname, nodeid, cldb)
197 if cache:
198 # try the cache
199 cache = self.cache.setdefault(classname, {})
200 if cache.has_key(nodeid):
201 return cache[nodeid]
203 # get from the database and save in the cache
204 if db is None:
205 db = self.getclassdb(classname)
206 if not db.has_key(nodeid):
207 raise IndexError, "no such %s %s"%(classname, nodeid)
208 res = marshal.loads(db[nodeid])
209 if cache:
210 cache[nodeid] = res
211 return res
213 def hasnode(self, classname, nodeid, db=None):
214 ''' determine if the database has a given node
215 '''
216 if DEBUG:
217 print 'hasnode', (self, classname, nodeid, cldb)
218 # try the cache
219 cache = self.cache.setdefault(classname, {})
220 if cache.has_key(nodeid):
221 return 1
223 # not in the cache - check the database
224 if db is None:
225 db = self.getclassdb(classname)
226 res = db.has_key(nodeid)
227 return res
229 def countnodes(self, classname, db=None):
230 if DEBUG:
231 print 'countnodes', (self, classname, cldb)
232 # include the new nodes not saved to the DB yet
233 count = len(self.newnodes.get(classname, {}))
235 # and count those in the DB
236 if db is None:
237 db = self.getclassdb(classname)
238 count = count + len(db.keys())
239 return count
241 def getnodeids(self, classname, db=None):
242 if DEBUG:
243 print 'getnodeids', (self, classname, db)
244 # start off with the new nodes
245 res = self.newnodes.get(classname, {}).keys()
247 if db is None:
248 db = self.getclassdb(classname)
249 res = res + db.keys()
250 return res
253 #
254 # Files - special node properties
255 #
256 def filename(self, classname, nodeid, property=None):
257 '''Determine what the filename for the given node and optionally property is.
258 '''
259 # TODO: split into multiple files directories
260 if property:
261 return os.path.join(self.dir, 'files', '%s%s.%s'%(classname,
262 nodeid, property))
263 else:
264 # roundupdb.FileClass never specified the property name, so don't include it
265 return os.path.join(self.dir, 'files', '%s%s'%(classname,
266 nodeid))
268 def storefile(self, classname, nodeid, property, content):
269 '''Store the content of the file in the database. The property may be None, in
270 which case the filename does not indicate which property is being saved.
271 '''
272 name = self.filename(classname, nodeid, property)
273 open(name + '.tmp', 'wb').write(content)
274 self.transactions.append((self._doStoreFile, (name, )))
276 def getfile(self, classname, nodeid, property):
277 '''Store the content of the file in the database.
278 '''
279 filename = self.filename(classname, nodeid, property)
280 try:
281 return open(filename, 'rb').read()
282 except:
283 return open(filename+'.tmp', 'rb').read()
286 #
287 # Journal
288 #
289 def addjournal(self, classname, nodeid, action, params):
290 ''' Journal the Action
291 'action' may be:
293 'create' or 'set' -- 'params' is a dictionary of property values
294 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
295 'retire' -- 'params' is None
296 '''
297 if DEBUG:
298 print 'addjournal', (self, classname, nodeid, action, params)
299 self.transactions.append((self._doSaveJournal, (classname, nodeid,
300 action, params)))
302 def getjournal(self, classname, nodeid):
303 ''' get the journal for id
304 '''
305 if DEBUG:
306 print 'getjournal', (self, classname, nodeid)
307 # attempt to open the journal - in some rare cases, the journal may
308 # not exist
309 try:
310 db = self._opendb('journals.%s'%classname, 'r')
311 except anydbm.error, error:
312 if str(error) == "need 'c' or 'n' flag to open new db": return []
313 elif error.args[0] != 2: raise
314 return []
315 journal = marshal.loads(db[nodeid])
316 res = []
317 for entry in journal:
318 (nodeid, date_stamp, self.journaltag, action, params) = entry
319 date_obj = date.Date(date_stamp)
320 res.append((nodeid, date_obj, self.journaltag, action, params))
321 return res
323 def pack(self, pack_before):
324 ''' delete all journal entries before 'pack_before' '''
325 if DEBUG:
326 print 'packjournal', (self, pack_before)
328 pack_before = pack_before.get_tuple()
330 classes = self.getclasses()
332 # TODO: factor this out to method - we're already doing it in
333 # _opendb.
334 db_type = ''
335 path = os.path.join(os.getcwd(), self.dir, classes[0])
336 if os.path.exists(path):
337 db_type = whichdb.whichdb(path)
338 if not db_type:
339 raise hyperdb.DatabaseError, "Couldn't identify database type"
340 elif os.path.exists(path+'.db'):
341 db_type = 'dbm'
343 for classname in classes:
344 db_name = 'journals.%s'%classname
345 db = self._opendb(db_name, 'w')
347 for key in db.keys():
348 journal = marshal.loads(db[key])
349 l = []
350 for entry in journal:
351 (nodeid, date_stamp, self.journaltag, action,
352 params) = entry
353 if date_stamp > pack_before or action == 'create':
354 l.append(entry)
355 elif action == 'set':
356 # grab the last set entry to keep information on
357 # activity
358 last_set_entry = entry
359 date_stamp = last_set_entry[1]
360 # if the last set entry was made after the pack date
361 # then it is already in the list
362 if date_stamp < pack_before:
363 l.append(last_set_entry)
364 db[key] = marshal.dumps(l)
365 if db_type == 'gdbm':
366 db.reorganize()
367 db.close()
370 #
371 # Basic transaction support
372 #
373 def commit(self):
374 ''' Commit the current transactions.
375 '''
376 if DEBUG:
377 print 'commit', (self,)
378 # TODO: lock the DB
380 # keep a handle to all the database files opened
381 self.databases = {}
383 # now, do all the transactions
384 for method, args in self.transactions:
385 method(*args)
387 # now close all the database files
388 for db in self.databases.values():
389 db.close()
390 del self.databases
391 # TODO: unlock the DB
393 # all transactions committed, back to normal
394 self.cache = {}
395 self.dirtynodes = {}
396 self.newnodes = {}
397 self.transactions = []
399 def _doSaveNode(self, classname, nodeid, node):
400 if DEBUG:
401 print '_doSaveNode', (self, classname, nodeid, node)
403 # get the database handle
404 db_name = 'nodes.%s'%classname
405 if self.databases.has_key(db_name):
406 db = self.databases[db_name]
407 else:
408 db = self.databases[db_name] = self.getclassdb(classname, 'c')
410 # now save the marshalled data
411 db[nodeid] = marshal.dumps(node)
413 def _doSaveJournal(self, classname, nodeid, action, params):
414 if DEBUG:
415 print '_doSaveJournal', (self, classname, nodeid, action, params)
416 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
417 params)
419 # get the database handle
420 db_name = 'journals.%s'%classname
421 if self.databases.has_key(db_name):
422 db = self.databases[db_name]
423 else:
424 db = self.databases[db_name] = self._opendb(db_name, 'c')
426 # now insert the journal entry
427 if db.has_key(nodeid):
428 s = db[nodeid]
429 l = marshal.loads(db[nodeid])
430 l.append(entry)
431 else:
432 l = [entry]
433 db[nodeid] = marshal.dumps(l)
435 def _doStoreFile(self, name, **databases):
436 # the file is currently ".tmp" - move it to its real name to commit
437 os.rename(name+".tmp", name)
439 def rollback(self):
440 ''' Reverse all actions from the current transaction.
441 '''
442 if DEBUG:
443 print 'rollback', (self, )
444 for method, args in self.transactions:
445 # delete temporary files
446 if method == self._doStoreFile:
447 if os.path.exists(args[0]+".tmp"):
448 os.remove(args[0]+".tmp")
449 self.cache = {}
450 self.dirtynodes = {}
451 self.newnodes = {}
452 self.transactions = []
454 #
455 #$Log: not supported by cvs2svn $
456 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
457 #You can now use the roundup-admin tool to pack the database
458 #
459 #Revision 1.23 2002/01/18 04:32:04 richard
460 #Rollback was breaking because a message hadn't actually been written to the file. Needs
461 #more investigation.
462 #
463 #Revision 1.22 2002/01/14 02:20:15 richard
464 # . changed all config accesses so they access either the instance or the
465 # config attriubute on the db. This means that all config is obtained from
466 # instance_config instead of the mish-mash of classes. This will make
467 # switching to a ConfigParser setup easier too, I hope.
468 #
469 #At a minimum, this makes migration a _little_ easier (a lot easier in the
470 #0.5.0 switch, I hope!)
471 #
472 #Revision 1.21 2002/01/02 02:31:38 richard
473 #Sorry for the huge checkin message - I was only intending to implement #496356
474 #but I found a number of places where things had been broken by transactions:
475 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
476 # for _all_ roundup-generated smtp messages to be sent to.
477 # . the transaction cache had broken the roundupdb.Class set() reactors
478 # . newly-created author users in the mailgw weren't being committed to the db
479 #
480 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
481 #on when I found that stuff :):
482 # . #496356 ] Use threading in messages
483 # . detectors were being registered multiple times
484 # . added tests for mailgw
485 # . much better attaching of erroneous messages in the mail gateway
486 #
487 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
488 #Fixed bugs:
489 # . Fixed file creation and retrieval in same transaction in anydbm
490 # backend
491 # . Cgi interface now renders new issue after issue creation
492 # . Could not set issue status to resolved through cgi interface
493 # . Mail gateway was changing status back to 'chatting' if status was
494 # omitted as an argument
495 #
496 #Revision 1.19 2001/12/17 03:52:48 richard
497 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
498 #storing more than one file per node - if a property name is supplied,
499 #the file is called designator.property.
500 #I decided not to migrate the existing files stored over to the new naming
501 #scheme - the FileClass just doesn't specify the property name.
502 #
503 #Revision 1.18 2001/12/16 10:53:38 richard
504 #take a copy of the node dict so that the subsequent set
505 #operation doesn't modify the oldvalues structure
506 #
507 #Revision 1.17 2001/12/14 23:42:57 richard
508 #yuck, a gdbm instance tests false :(
509 #I've left the debugging code in - it should be removed one day if we're ever
510 #_really_ anal about performace :)
511 #
512 #Revision 1.16 2001/12/12 03:23:14 richard
513 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
514 #incorrectly identifies a dbm file as a dbhash file on my system. This has
515 #been submitted to the python bug tracker as issue #491888:
516 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
517 #
518 #Revision 1.15 2001/12/12 02:30:51 richard
519 #I fixed the problems with people whose anydbm was using the dbm module at the
520 #backend. It turns out the dbm module modifies the file name to append ".db"
521 #and my check to determine if we're opening an existing or new db just
522 #tested os.path.exists() on the filename. Well, no longer! We now perform a
523 #much better check _and_ cope with the anydbm implementation module changing
524 #too!
525 #I also fixed the backends __init__ so only ImportError is squashed.
526 #
527 #Revision 1.14 2001/12/10 22:20:01 richard
528 #Enabled transaction support in the bsddb backend. It uses the anydbm code
529 #where possible, only replacing methods where the db is opened (it uses the
530 #btree opener specifically.)
531 #Also cleaned up some change note generation.
532 #Made the backends package work with pydoc too.
533 #
534 #Revision 1.13 2001/12/02 05:06:16 richard
535 #. We now use weakrefs in the Classes to keep the database reference, so
536 # the close() method on the database is no longer needed.
537 # I bumped the minimum python requirement up to 2.1 accordingly.
538 #. #487480 ] roundup-server
539 #. #487476 ] INSTALL.txt
540 #
541 #I also cleaned up the change message / post-edit stuff in the cgi client.
542 #There's now a clearly marked "TODO: append the change note" where I believe
543 #the change note should be added there. The "changes" list will obviously
544 #have to be modified to be a dict of the changes, or somesuch.
545 #
546 #More testing needed.
547 #
548 #Revision 1.12 2001/12/01 07:17:50 richard
549 #. We now have basic transaction support! Information is only written to
550 # the database when the commit() method is called. Only the anydbm
551 # backend is modified in this way - neither of the bsddb backends have been.
552 # The mail, admin and cgi interfaces all use commit (except the admin tool
553 # doesn't have a commit command, so interactive users can't commit...)
554 #. Fixed login/registration forwarding the user to the right page (or not,
555 # on a failure)
556 #
557 #Revision 1.11 2001/11/21 02:34:18 richard
558 #Added a target version field to the extended issue schema
559 #
560 #Revision 1.10 2001/10/09 23:58:10 richard
561 #Moved the data stringification up into the hyperdb.Class class' get, set
562 #and create methods. This means that the data is also stringified for the
563 #journal call, and removes duplication of code from the backends. The
564 #backend code now only sees strings.
565 #
566 #Revision 1.9 2001/10/09 07:25:59 richard
567 #Added the Password property type. See "pydoc roundup.password" for
568 #implementation details. Have updated some of the documentation too.
569 #
570 #Revision 1.8 2001/09/29 13:27:00 richard
571 #CGI interfaces now spit up a top-level index of all the instances they can
572 #serve.
573 #
574 #Revision 1.7 2001/08/12 06:32:36 richard
575 #using isinstance(blah, Foo) now instead of isFooType
576 #
577 #Revision 1.6 2001/08/07 00:24:42 richard
578 #stupid typo
579 #
580 #Revision 1.5 2001/08/07 00:15:51 richard
581 #Added the copyright/license notice to (nearly) all files at request of
582 #Bizar Software.
583 #
584 #Revision 1.4 2001/07/30 01:41:36 richard
585 #Makes schema changes mucho easier.
586 #
587 #Revision 1.3 2001/07/25 01:23:07 richard
588 #Added the Roundup spec to the new documentation directory.
589 #
590 #Revision 1.2 2001/07/23 08:20:44 richard
591 #Moved over to using marshal in the bsddb and anydbm backends.
592 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
593 # retired - mod hyperdb.Class.list() so it lists retired nodes)
594 #
595 #