4735d04364299f6a80495b855e0e2431eaaf84a2
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.32 2002-04-15 23:25:15 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal
27 from roundup import hyperdb, date
28 from blobfiles import FileStorage
29 from locking import acquire_lock, release_lock
31 #
32 # Now the database
33 #
34 class Database(FileStorage, hyperdb.Database):
35 """A database for storing records containing flexible data types.
37 Transaction stuff TODO:
38 . check the timestamp of the class file and nuke the cache if it's
39 modified. Do some sort of conflict checking on the dirty stuff.
40 . perhaps detect write collisions (related to above)?
42 """
43 def __init__(self, config, journaltag=None):
44 """Open a hyperdatabase given a specifier to some storage.
46 The 'storagelocator' is obtained from config.DATABASE.
47 The meaning of 'storagelocator' depends on the particular
48 implementation of the hyperdatabase. It could be a file name,
49 a directory path, a socket descriptor for a connection to a
50 database over the network, etc.
52 The 'journaltag' is a token that will be attached to the journal
53 entries for any edits done on the database. If 'journaltag' is
54 None, the database is opened in read-only mode: the Class.create(),
55 Class.set(), and Class.retire() methods are disabled.
56 """
57 self.config, self.journaltag = config, journaltag
58 self.dir = config.DATABASE
59 self.classes = {}
60 self.cache = {} # cache of nodes loaded or created
61 self.dirtynodes = {} # keep track of the dirty nodes by class
62 self.newnodes = {} # keep track of the new nodes by class
63 self.transactions = []
65 def __repr__(self):
66 return '<back_anydbm instance at %x>'%id(self)
68 #
69 # Classes
70 #
71 def __getattr__(self, classname):
72 """A convenient way of calling self.getclass(classname)."""
73 if self.classes.has_key(classname):
74 if hyperdb.DEBUG:
75 print '__getattr__', (self, classname)
76 return self.classes[classname]
77 raise AttributeError, classname
79 def addclass(self, cl):
80 if hyperdb.DEBUG:
81 print 'addclass', (self, cl)
82 cn = cl.classname
83 if self.classes.has_key(cn):
84 raise ValueError, cn
85 self.classes[cn] = cl
87 def getclasses(self):
88 """Return a list of the names of all existing classes."""
89 if hyperdb.DEBUG:
90 print 'getclasses', (self,)
91 l = self.classes.keys()
92 l.sort()
93 return l
95 def getclass(self, classname):
96 """Get the Class object representing a particular class.
98 If 'classname' is not a valid class name, a KeyError is raised.
99 """
100 if hyperdb.DEBUG:
101 print 'getclass', (self, classname)
102 return self.classes[classname]
104 #
105 # Class DBs
106 #
107 def clear(self):
108 '''Delete all database contents
109 '''
110 if hyperdb.DEBUG:
111 print 'clear', (self,)
112 for cn in self.classes.keys():
113 for dummy in 'nodes', 'journals':
114 path = os.path.join(self.dir, 'journals.%s'%cn)
115 if os.path.exists(path):
116 os.remove(path)
117 elif os.path.exists(path+'.db'): # dbm appends .db
118 os.remove(path+'.db')
120 def getclassdb(self, classname, mode='r'):
121 ''' grab a connection to the class db that will be used for
122 multiple actions
123 '''
124 if hyperdb.DEBUG:
125 print 'getclassdb', (self, classname, mode)
126 return self._opendb('nodes.%s'%classname, mode)
128 def _opendb(self, name, mode):
129 '''Low-level database opener that gets around anydbm/dbm
130 eccentricities.
131 '''
132 if hyperdb.DEBUG:
133 print '_opendb', (self, name, mode)
135 # determine which DB wrote the class file
136 db_type = ''
137 path = os.path.join(os.getcwd(), self.dir, name)
138 if os.path.exists(path):
139 db_type = whichdb.whichdb(path)
140 if not db_type:
141 raise hyperdb.DatabaseError, "Couldn't identify database type"
142 elif os.path.exists(path+'.db'):
143 # if the path ends in '.db', it's a dbm database, whether
144 # anydbm says it's dbhash or not!
145 db_type = 'dbm'
147 # new database? let anydbm pick the best dbm
148 if not db_type:
149 if hyperdb.DEBUG:
150 print "_opendb anydbm.open(%r, 'n')"%path
151 return anydbm.open(path, 'n')
153 # open the database with the correct module
154 try:
155 dbm = __import__(db_type)
156 except ImportError:
157 raise hyperdb.DatabaseError, \
158 "Couldn't open database - the required module '%s'"\
159 "is not available"%db_type
160 if hyperdb.DEBUG:
161 print "_opendb %r.open(%r, %r)"%(db_type, path, mode)
162 return dbm.open(path, mode)
164 def _lockdb(self, name):
165 ''' Lock a database file
166 '''
167 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
168 return acquire_lock(path)
170 #
171 # Node IDs
172 #
173 def newid(self, classname):
174 ''' Generate a new id for the given class
175 '''
176 # open the ids DB - create if if doesn't exist
177 lock = self._lockdb('_ids')
178 db = self._opendb('_ids', 'c')
179 if db.has_key(classname):
180 newid = db[classname] = str(int(db[classname]) + 1)
181 else:
182 # the count() bit is transitional - older dbs won't start at 1
183 newid = str(self.getclass(classname).count()+1)
184 db[classname] = newid
185 db.close()
186 release_lock(lock)
187 return newid
189 #
190 # Nodes
191 #
192 def addnode(self, classname, nodeid, node):
193 ''' add the specified node to its class's db
194 '''
195 if hyperdb.DEBUG:
196 print 'addnode', (self, classname, nodeid, node)
197 self.newnodes.setdefault(classname, {})[nodeid] = 1
198 self.cache.setdefault(classname, {})[nodeid] = node
199 self.savenode(classname, nodeid, node)
201 def setnode(self, classname, nodeid, node):
202 ''' change the specified node
203 '''
204 if hyperdb.DEBUG:
205 print 'setnode', (self, classname, nodeid, node)
206 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
208 # can't set without having already loaded the node
209 self.cache[classname][nodeid] = node
210 self.savenode(classname, nodeid, node)
212 def savenode(self, classname, nodeid, node):
213 ''' perform the saving of data specified by the set/addnode
214 '''
215 if hyperdb.DEBUG:
216 print 'savenode', (self, classname, nodeid, node)
217 self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
219 def getnode(self, classname, nodeid, db=None, cache=1):
220 ''' get a node from the database
221 '''
222 if hyperdb.DEBUG:
223 print 'getnode', (self, classname, nodeid, db)
224 if cache:
225 # try the cache
226 cache = self.cache.setdefault(classname, {})
227 if cache.has_key(nodeid):
228 return cache[nodeid]
230 # get from the database and save in the cache
231 if db is None:
232 db = self.getclassdb(classname)
233 if not db.has_key(nodeid):
234 raise IndexError, "no such %s %s"%(classname, nodeid)
236 # decode
237 res = marshal.loads(db[nodeid])
239 # reverse the serialisation
240 res = self.unserialise(classname, res)
242 # store off in the cache
243 if cache:
244 cache[nodeid] = res
246 return res
248 def hasnode(self, classname, nodeid, db=None):
249 ''' determine if the database has a given node
250 '''
251 if hyperdb.DEBUG:
252 print 'hasnode', (self, classname, nodeid, db)
253 # try the cache
254 cache = self.cache.setdefault(classname, {})
255 if cache.has_key(nodeid):
256 return 1
258 # not in the cache - check the database
259 if db is None:
260 db = self.getclassdb(classname)
261 res = db.has_key(nodeid)
262 return res
264 def countnodes(self, classname, db=None):
265 if hyperdb.DEBUG:
266 print 'countnodes', (self, classname, db)
267 # include the new nodes not saved to the DB yet
268 count = len(self.newnodes.get(classname, {}))
270 # and count those in the DB
271 if db is None:
272 db = self.getclassdb(classname)
273 count = count + len(db.keys())
274 return count
276 def getnodeids(self, classname, db=None):
277 if hyperdb.DEBUG:
278 print 'getnodeids', (self, classname, db)
279 # start off with the new nodes
280 res = self.newnodes.get(classname, {}).keys()
282 if db is None:
283 db = self.getclassdb(classname)
284 res = res + db.keys()
285 return res
288 #
289 # Files - special node properties
290 # inherited from FileStorage
292 #
293 # Journal
294 #
295 def addjournal(self, classname, nodeid, action, params):
296 ''' Journal the Action
297 'action' may be:
299 'create' or 'set' -- 'params' is a dictionary of property values
300 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
301 'retire' -- 'params' is None
302 '''
303 if hyperdb.DEBUG:
304 print 'addjournal', (self, classname, nodeid, action, params)
305 self.transactions.append((self._doSaveJournal, (classname, nodeid,
306 action, params)))
308 def getjournal(self, classname, nodeid):
309 ''' get the journal for id
310 '''
311 if hyperdb.DEBUG:
312 print 'getjournal', (self, classname, nodeid)
313 # attempt to open the journal - in some rare cases, the journal may
314 # not exist
315 try:
316 db = self._opendb('journals.%s'%classname, 'r')
317 except anydbm.error, error:
318 if str(error) == "need 'c' or 'n' flag to open new db": return []
319 elif error.args[0] != 2: raise
320 return []
321 journal = marshal.loads(db[nodeid])
322 res = []
323 for entry in journal:
324 (nodeid, date_stamp, user, action, params) = entry
325 date_obj = date.Date(date_stamp)
326 res.append((nodeid, date_obj, user, action, params))
327 return res
329 def pack(self, pack_before):
330 ''' delete all journal entries before 'pack_before' '''
331 if hyperdb.DEBUG:
332 print 'packjournal', (self, pack_before)
334 pack_before = pack_before.get_tuple()
336 classes = self.getclasses()
338 # TODO: factor this out to method - we're already doing it in
339 # _opendb.
340 db_type = ''
341 path = os.path.join(os.getcwd(), self.dir, classes[0])
342 if os.path.exists(path):
343 db_type = whichdb.whichdb(path)
344 if not db_type:
345 raise hyperdb.DatabaseError, "Couldn't identify database type"
346 elif os.path.exists(path+'.db'):
347 db_type = 'dbm'
349 for classname in classes:
350 db_name = 'journals.%s'%classname
351 db = self._opendb(db_name, 'w')
353 for key in db.keys():
354 journal = marshal.loads(db[key])
355 l = []
356 last_set_entry = None
357 for entry in journal:
358 (nodeid, date_stamp, self.journaltag, action,
359 params) = entry
360 if date_stamp > pack_before or action == 'create':
361 l.append(entry)
362 elif action == 'set':
363 # grab the last set entry to keep information on
364 # activity
365 last_set_entry = entry
366 if last_set_entry:
367 date_stamp = last_set_entry[1]
368 # if the last set entry was made after the pack date
369 # then it is already in the list
370 if date_stamp < pack_before:
371 l.append(last_set_entry)
372 db[key] = marshal.dumps(l)
373 if db_type == 'gdbm':
374 db.reorganize()
375 db.close()
378 #
379 # Basic transaction support
380 #
381 def commit(self):
382 ''' Commit the current transactions.
383 '''
384 if hyperdb.DEBUG:
385 print 'commit', (self,)
386 # TODO: lock the DB
388 # keep a handle to all the database files opened
389 self.databases = {}
391 # now, do all the transactions
392 for method, args in self.transactions:
393 method(*args)
395 # now close all the database files
396 for db in self.databases.values():
397 db.close()
398 del self.databases
399 # TODO: unlock the DB
401 # all transactions committed, back to normal
402 self.cache = {}
403 self.dirtynodes = {}
404 self.newnodes = {}
405 self.transactions = []
407 def _doSaveNode(self, classname, nodeid, node):
408 if hyperdb.DEBUG:
409 print '_doSaveNode', (self, classname, nodeid, node)
411 # get the database handle
412 db_name = 'nodes.%s'%classname
413 if self.databases.has_key(db_name):
414 db = self.databases[db_name]
415 else:
416 db = self.databases[db_name] = self.getclassdb(classname, 'c')
418 # now save the marshalled data
419 db[nodeid] = marshal.dumps(self.serialise(classname, node))
421 def _doSaveJournal(self, classname, nodeid, action, params):
422 # serialise first
423 if action in ('set', 'create'):
424 params = self.serialise(classname, params)
426 # create the journal entry
427 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
428 params)
430 if hyperdb.DEBUG:
431 print '_doSaveJournal', entry
433 # get the database handle
434 db_name = 'journals.%s'%classname
435 if self.databases.has_key(db_name):
436 db = self.databases[db_name]
437 else:
438 db = self.databases[db_name] = self._opendb(db_name, 'c')
440 # now insert the journal entry
441 if db.has_key(nodeid):
442 # append to existing
443 s = db[nodeid]
444 l = marshal.loads(s)
445 l.append(entry)
446 else:
447 l = [entry]
449 db[nodeid] = marshal.dumps(l)
451 def _doStoreFile(self, name, **databases):
452 # the file is currently ".tmp" - move it to its real name to commit
453 os.rename(name+".tmp", name)
455 def rollback(self):
456 ''' Reverse all actions from the current transaction.
457 '''
458 if hyperdb.DEBUG:
459 print 'rollback', (self, )
460 for method, args in self.transactions:
461 # delete temporary files
462 if method == self._doStoreFile:
463 if os.path.exists(args[0]+".tmp"):
464 os.remove(args[0]+".tmp")
465 self.cache = {}
466 self.dirtynodes = {}
467 self.newnodes = {}
468 self.transactions = []
470 #
471 #$Log: not supported by cvs2svn $
472 #Revision 1.31 2002/04/03 05:54:31 richard
473 #Fixed serialisation problem by moving the serialisation step out of the
474 #hyperdb.Class (get, set) into the hyperdb.Database.
475 #
476 #Also fixed htmltemplate after the showid changes I made yesterday.
477 #
478 #Unit tests for all of the above written.
479 #
480 #Revision 1.30 2002/02/27 03:40:59 richard
481 #Ran it through pychecker, made fixes
482 #
483 #Revision 1.29 2002/02/25 14:34:31 grubert
484 # . use blobfiles in back_anydbm which is used in back_bsddb.
485 # change test_db as dirlist does not work for subdirectories.
486 # ATTENTION: blobfiles now creates subdirectories for files.
487 #
488 #Revision 1.28 2002/02/16 09:14:17 richard
489 # . #514854 ] History: "User" is always ticket creator
490 #
491 #Revision 1.27 2002/01/22 07:21:13 richard
492 #. fixed back_bsddb so it passed the journal tests
493 #
494 #... it didn't seem happy using the back_anydbm _open method, which is odd.
495 #Yet another occurrance of whichdb not being able to recognise older bsddb
496 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
497 #process.
498 #
499 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
500 #last_set_entry was referenced before assignment
501 #
502 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
503 #We need to keep the last 'set' entry in the journal to preserve
504 #information on 'activity' for nodes.
505 #
506 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
507 #You can now use the roundup-admin tool to pack the database
508 #
509 #Revision 1.23 2002/01/18 04:32:04 richard
510 #Rollback was breaking because a message hadn't actually been written to the file. Needs
511 #more investigation.
512 #
513 #Revision 1.22 2002/01/14 02:20:15 richard
514 # . changed all config accesses so they access either the instance or the
515 # config attriubute on the db. This means that all config is obtained from
516 # instance_config instead of the mish-mash of classes. This will make
517 # switching to a ConfigParser setup easier too, I hope.
518 #
519 #At a minimum, this makes migration a _little_ easier (a lot easier in the
520 #0.5.0 switch, I hope!)
521 #
522 #Revision 1.21 2002/01/02 02:31:38 richard
523 #Sorry for the huge checkin message - I was only intending to implement #496356
524 #but I found a number of places where things had been broken by transactions:
525 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
526 # for _all_ roundup-generated smtp messages to be sent to.
527 # . the transaction cache had broken the roundupdb.Class set() reactors
528 # . newly-created author users in the mailgw weren't being committed to the db
529 #
530 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
531 #on when I found that stuff :):
532 # . #496356 ] Use threading in messages
533 # . detectors were being registered multiple times
534 # . added tests for mailgw
535 # . much better attaching of erroneous messages in the mail gateway
536 #
537 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
538 #Fixed bugs:
539 # . Fixed file creation and retrieval in same transaction in anydbm
540 # backend
541 # . Cgi interface now renders new issue after issue creation
542 # . Could not set issue status to resolved through cgi interface
543 # . Mail gateway was changing status back to 'chatting' if status was
544 # omitted as an argument
545 #
546 #Revision 1.19 2001/12/17 03:52:48 richard
547 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
548 #storing more than one file per node - if a property name is supplied,
549 #the file is called designator.property.
550 #I decided not to migrate the existing files stored over to the new naming
551 #scheme - the FileClass just doesn't specify the property name.
552 #
553 #Revision 1.18 2001/12/16 10:53:38 richard
554 #take a copy of the node dict so that the subsequent set
555 #operation doesn't modify the oldvalues structure
556 #
557 #Revision 1.17 2001/12/14 23:42:57 richard
558 #yuck, a gdbm instance tests false :(
559 #I've left the debugging code in - it should be removed one day if we're ever
560 #_really_ anal about performace :)
561 #
562 #Revision 1.16 2001/12/12 03:23:14 richard
563 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
564 #incorrectly identifies a dbm file as a dbhash file on my system. This has
565 #been submitted to the python bug tracker as issue #491888:
566 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
567 #
568 #Revision 1.15 2001/12/12 02:30:51 richard
569 #I fixed the problems with people whose anydbm was using the dbm module at the
570 #backend. It turns out the dbm module modifies the file name to append ".db"
571 #and my check to determine if we're opening an existing or new db just
572 #tested os.path.exists() on the filename. Well, no longer! We now perform a
573 #much better check _and_ cope with the anydbm implementation module changing
574 #too!
575 #I also fixed the backends __init__ so only ImportError is squashed.
576 #
577 #Revision 1.14 2001/12/10 22:20:01 richard
578 #Enabled transaction support in the bsddb backend. It uses the anydbm code
579 #where possible, only replacing methods where the db is opened (it uses the
580 #btree opener specifically.)
581 #Also cleaned up some change note generation.
582 #Made the backends package work with pydoc too.
583 #
584 #Revision 1.13 2001/12/02 05:06:16 richard
585 #. We now use weakrefs in the Classes to keep the database reference, so
586 # the close() method on the database is no longer needed.
587 # I bumped the minimum python requirement up to 2.1 accordingly.
588 #. #487480 ] roundup-server
589 #. #487476 ] INSTALL.txt
590 #
591 #I also cleaned up the change message / post-edit stuff in the cgi client.
592 #There's now a clearly marked "TODO: append the change note" where I believe
593 #the change note should be added there. The "changes" list will obviously
594 #have to be modified to be a dict of the changes, or somesuch.
595 #
596 #More testing needed.
597 #
598 #Revision 1.12 2001/12/01 07:17:50 richard
599 #. We now have basic transaction support! Information is only written to
600 # the database when the commit() method is called. Only the anydbm
601 # backend is modified in this way - neither of the bsddb backends have been.
602 # The mail, admin and cgi interfaces all use commit (except the admin tool
603 # doesn't have a commit command, so interactive users can't commit...)
604 #. Fixed login/registration forwarding the user to the right page (or not,
605 # on a failure)
606 #
607 #Revision 1.11 2001/11/21 02:34:18 richard
608 #Added a target version field to the extended issue schema
609 #
610 #Revision 1.10 2001/10/09 23:58:10 richard
611 #Moved the data stringification up into the hyperdb.Class class' get, set
612 #and create methods. This means that the data is also stringified for the
613 #journal call, and removes duplication of code from the backends. The
614 #backend code now only sees strings.
615 #
616 #Revision 1.9 2001/10/09 07:25:59 richard
617 #Added the Password property type. See "pydoc roundup.password" for
618 #implementation details. Have updated some of the documentation too.
619 #
620 #Revision 1.8 2001/09/29 13:27:00 richard
621 #CGI interfaces now spit up a top-level index of all the instances they can
622 #serve.
623 #
624 #Revision 1.7 2001/08/12 06:32:36 richard
625 #using isinstance(blah, Foo) now instead of isFooType
626 #
627 #Revision 1.6 2001/08/07 00:24:42 richard
628 #stupid typo
629 #
630 #Revision 1.5 2001/08/07 00:15:51 richard
631 #Added the copyright/license notice to (nearly) all files at request of
632 #Bizar Software.
633 #
634 #Revision 1.4 2001/07/30 01:41:36 richard
635 #Makes schema changes mucho easier.
636 #
637 #Revision 1.3 2001/07/25 01:23:07 richard
638 #Added the Roundup spec to the new documentation directory.
639 #
640 #Revision 1.2 2001/07/23 08:20:44 richard
641 #Moved over to using marshal in the bsddb and anydbm backends.
642 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
643 # retired - mod hyperdb.Class.list() so it lists retired nodes)
644 #
645 #