be4bd4ef639ad4603a555e117a4df405a96bea7d
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.33 2002-04-24 10:38:26 rochecompaan Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal
27 from roundup import hyperdb, date
28 from blobfiles import FileStorage
29 from locking import acquire_lock, release_lock
31 #
32 # Now the database
33 #
34 class Database(FileStorage, hyperdb.Database):
35 """A database for storing records containing flexible data types.
37 Transaction stuff TODO:
38 . check the timestamp of the class file and nuke the cache if it's
39 modified. Do some sort of conflict checking on the dirty stuff.
40 . perhaps detect write collisions (related to above)?
42 """
43 def __init__(self, config, journaltag=None):
44 """Open a hyperdatabase given a specifier to some storage.
46 The 'storagelocator' is obtained from config.DATABASE.
47 The meaning of 'storagelocator' depends on the particular
48 implementation of the hyperdatabase. It could be a file name,
49 a directory path, a socket descriptor for a connection to a
50 database over the network, etc.
52 The 'journaltag' is a token that will be attached to the journal
53 entries for any edits done on the database. If 'journaltag' is
54 None, the database is opened in read-only mode: the Class.create(),
55 Class.set(), and Class.retire() methods are disabled.
56 """
57 self.config, self.journaltag = config, journaltag
58 self.dir = config.DATABASE
59 self.classes = {}
60 self.cache = {} # cache of nodes loaded or created
61 self.dirtynodes = {} # keep track of the dirty nodes by class
62 self.newnodes = {} # keep track of the new nodes by class
63 self.transactions = []
64 # ensure files are group readable and writable
65 os.umask(0002)
67 def __repr__(self):
68 return '<back_anydbm instance at %x>'%id(self)
70 #
71 # Classes
72 #
73 def __getattr__(self, classname):
74 """A convenient way of calling self.getclass(classname)."""
75 if self.classes.has_key(classname):
76 if hyperdb.DEBUG:
77 print '__getattr__', (self, classname)
78 return self.classes[classname]
79 raise AttributeError, classname
81 def addclass(self, cl):
82 if hyperdb.DEBUG:
83 print 'addclass', (self, cl)
84 cn = cl.classname
85 if self.classes.has_key(cn):
86 raise ValueError, cn
87 self.classes[cn] = cl
89 def getclasses(self):
90 """Return a list of the names of all existing classes."""
91 if hyperdb.DEBUG:
92 print 'getclasses', (self,)
93 l = self.classes.keys()
94 l.sort()
95 return l
97 def getclass(self, classname):
98 """Get the Class object representing a particular class.
100 If 'classname' is not a valid class name, a KeyError is raised.
101 """
102 if hyperdb.DEBUG:
103 print 'getclass', (self, classname)
104 return self.classes[classname]
106 #
107 # Class DBs
108 #
109 def clear(self):
110 '''Delete all database contents
111 '''
112 if hyperdb.DEBUG:
113 print 'clear', (self,)
114 for cn in self.classes.keys():
115 for dummy in 'nodes', 'journals':
116 path = os.path.join(self.dir, 'journals.%s'%cn)
117 if os.path.exists(path):
118 os.remove(path)
119 elif os.path.exists(path+'.db'): # dbm appends .db
120 os.remove(path+'.db')
122 def getclassdb(self, classname, mode='r'):
123 ''' grab a connection to the class db that will be used for
124 multiple actions
125 '''
126 if hyperdb.DEBUG:
127 print 'getclassdb', (self, classname, mode)
128 return self._opendb('nodes.%s'%classname, mode)
130 def _opendb(self, name, mode):
131 '''Low-level database opener that gets around anydbm/dbm
132 eccentricities.
133 '''
134 if hyperdb.DEBUG:
135 print '_opendb', (self, name, mode)
137 # determine which DB wrote the class file
138 db_type = ''
139 path = os.path.join(os.getcwd(), self.dir, name)
140 if os.path.exists(path):
141 db_type = whichdb.whichdb(path)
142 if not db_type:
143 raise hyperdb.DatabaseError, "Couldn't identify database type"
144 elif os.path.exists(path+'.db'):
145 # if the path ends in '.db', it's a dbm database, whether
146 # anydbm says it's dbhash or not!
147 db_type = 'dbm'
149 # new database? let anydbm pick the best dbm
150 if not db_type:
151 if hyperdb.DEBUG:
152 print "_opendb anydbm.open(%r, 'n')"%path
153 return anydbm.open(path, 'n')
155 # open the database with the correct module
156 try:
157 dbm = __import__(db_type)
158 except ImportError:
159 raise hyperdb.DatabaseError, \
160 "Couldn't open database - the required module '%s'"\
161 "is not available"%db_type
162 if hyperdb.DEBUG:
163 print "_opendb %r.open(%r, %r)"%(db_type, path, mode)
164 return dbm.open(path, mode)
166 def _lockdb(self, name):
167 ''' Lock a database file
168 '''
169 path = os.path.join(os.getcwd(), self.dir, '%s.lock'%name)
170 return acquire_lock(path)
172 #
173 # Node IDs
174 #
175 def newid(self, classname):
176 ''' Generate a new id for the given class
177 '''
178 # open the ids DB - create if if doesn't exist
179 lock = self._lockdb('_ids')
180 db = self._opendb('_ids', 'c')
181 if db.has_key(classname):
182 newid = db[classname] = str(int(db[classname]) + 1)
183 else:
184 # the count() bit is transitional - older dbs won't start at 1
185 newid = str(self.getclass(classname).count()+1)
186 db[classname] = newid
187 db.close()
188 release_lock(lock)
189 return newid
191 #
192 # Nodes
193 #
194 def addnode(self, classname, nodeid, node):
195 ''' add the specified node to its class's db
196 '''
197 if hyperdb.DEBUG:
198 print 'addnode', (self, classname, nodeid, node)
199 self.newnodes.setdefault(classname, {})[nodeid] = 1
200 self.cache.setdefault(classname, {})[nodeid] = node
201 self.savenode(classname, nodeid, node)
203 def setnode(self, classname, nodeid, node):
204 ''' change the specified node
205 '''
206 if hyperdb.DEBUG:
207 print 'setnode', (self, classname, nodeid, node)
208 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
210 # can't set without having already loaded the node
211 self.cache[classname][nodeid] = node
212 self.savenode(classname, nodeid, node)
214 def savenode(self, classname, nodeid, node):
215 ''' perform the saving of data specified by the set/addnode
216 '''
217 if hyperdb.DEBUG:
218 print 'savenode', (self, classname, nodeid, node)
219 self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
221 def getnode(self, classname, nodeid, db=None, cache=1):
222 ''' get a node from the database
223 '''
224 if hyperdb.DEBUG:
225 print 'getnode', (self, classname, nodeid, db)
226 if cache:
227 # try the cache
228 cache = self.cache.setdefault(classname, {})
229 if cache.has_key(nodeid):
230 return cache[nodeid]
232 # get from the database and save in the cache
233 if db is None:
234 db = self.getclassdb(classname)
235 if not db.has_key(nodeid):
236 raise IndexError, "no such %s %s"%(classname, nodeid)
238 # decode
239 res = marshal.loads(db[nodeid])
241 # reverse the serialisation
242 res = self.unserialise(classname, res)
244 # store off in the cache
245 if cache:
246 cache[nodeid] = res
248 return res
250 def hasnode(self, classname, nodeid, db=None):
251 ''' determine if the database has a given node
252 '''
253 if hyperdb.DEBUG:
254 print 'hasnode', (self, classname, nodeid, db)
255 # try the cache
256 cache = self.cache.setdefault(classname, {})
257 if cache.has_key(nodeid):
258 return 1
260 # not in the cache - check the database
261 if db is None:
262 db = self.getclassdb(classname)
263 res = db.has_key(nodeid)
264 return res
266 def countnodes(self, classname, db=None):
267 if hyperdb.DEBUG:
268 print 'countnodes', (self, classname, db)
269 # include the new nodes not saved to the DB yet
270 count = len(self.newnodes.get(classname, {}))
272 # and count those in the DB
273 if db is None:
274 db = self.getclassdb(classname)
275 count = count + len(db.keys())
276 return count
278 def getnodeids(self, classname, db=None):
279 if hyperdb.DEBUG:
280 print 'getnodeids', (self, classname, db)
281 # start off with the new nodes
282 res = self.newnodes.get(classname, {}).keys()
284 if db is None:
285 db = self.getclassdb(classname)
286 res = res + db.keys()
287 return res
290 #
291 # Files - special node properties
292 # inherited from FileStorage
294 #
295 # Journal
296 #
297 def addjournal(self, classname, nodeid, action, params):
298 ''' Journal the Action
299 'action' may be:
301 'create' or 'set' -- 'params' is a dictionary of property values
302 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
303 'retire' -- 'params' is None
304 '''
305 if hyperdb.DEBUG:
306 print 'addjournal', (self, classname, nodeid, action, params)
307 self.transactions.append((self._doSaveJournal, (classname, nodeid,
308 action, params)))
310 def getjournal(self, classname, nodeid):
311 ''' get the journal for id
312 '''
313 if hyperdb.DEBUG:
314 print 'getjournal', (self, classname, nodeid)
315 # attempt to open the journal - in some rare cases, the journal may
316 # not exist
317 try:
318 db = self._opendb('journals.%s'%classname, 'r')
319 except anydbm.error, error:
320 if str(error) == "need 'c' or 'n' flag to open new db": return []
321 elif error.args[0] != 2: raise
322 return []
323 journal = marshal.loads(db[nodeid])
324 res = []
325 for entry in journal:
326 (nodeid, date_stamp, user, action, params) = entry
327 date_obj = date.Date(date_stamp)
328 res.append((nodeid, date_obj, user, action, params))
329 return res
331 def pack(self, pack_before):
332 ''' delete all journal entries before 'pack_before' '''
333 if hyperdb.DEBUG:
334 print 'packjournal', (self, pack_before)
336 pack_before = pack_before.get_tuple()
338 classes = self.getclasses()
340 # TODO: factor this out to method - we're already doing it in
341 # _opendb.
342 db_type = ''
343 path = os.path.join(os.getcwd(), self.dir, classes[0])
344 if os.path.exists(path):
345 db_type = whichdb.whichdb(path)
346 if not db_type:
347 raise hyperdb.DatabaseError, "Couldn't identify database type"
348 elif os.path.exists(path+'.db'):
349 db_type = 'dbm'
351 for classname in classes:
352 db_name = 'journals.%s'%classname
353 db = self._opendb(db_name, 'w')
355 for key in db.keys():
356 journal = marshal.loads(db[key])
357 l = []
358 last_set_entry = None
359 for entry in journal:
360 (nodeid, date_stamp, self.journaltag, action,
361 params) = entry
362 if date_stamp > pack_before or action == 'create':
363 l.append(entry)
364 elif action == 'set':
365 # grab the last set entry to keep information on
366 # activity
367 last_set_entry = entry
368 if last_set_entry:
369 date_stamp = last_set_entry[1]
370 # if the last set entry was made after the pack date
371 # then it is already in the list
372 if date_stamp < pack_before:
373 l.append(last_set_entry)
374 db[key] = marshal.dumps(l)
375 if db_type == 'gdbm':
376 db.reorganize()
377 db.close()
380 #
381 # Basic transaction support
382 #
383 def commit(self):
384 ''' Commit the current transactions.
385 '''
386 if hyperdb.DEBUG:
387 print 'commit', (self,)
388 # TODO: lock the DB
390 # keep a handle to all the database files opened
391 self.databases = {}
393 # now, do all the transactions
394 for method, args in self.transactions:
395 method(*args)
397 # now close all the database files
398 for db in self.databases.values():
399 db.close()
400 del self.databases
401 # TODO: unlock the DB
403 # all transactions committed, back to normal
404 self.cache = {}
405 self.dirtynodes = {}
406 self.newnodes = {}
407 self.transactions = []
409 def _doSaveNode(self, classname, nodeid, node):
410 if hyperdb.DEBUG:
411 print '_doSaveNode', (self, classname, nodeid, node)
413 # get the database handle
414 db_name = 'nodes.%s'%classname
415 if self.databases.has_key(db_name):
416 db = self.databases[db_name]
417 else:
418 db = self.databases[db_name] = self.getclassdb(classname, 'c')
420 # now save the marshalled data
421 db[nodeid] = marshal.dumps(self.serialise(classname, node))
423 def _doSaveJournal(self, classname, nodeid, action, params):
424 # serialise first
425 if action in ('set', 'create'):
426 params = self.serialise(classname, params)
428 # create the journal entry
429 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
430 params)
432 if hyperdb.DEBUG:
433 print '_doSaveJournal', entry
435 # get the database handle
436 db_name = 'journals.%s'%classname
437 if self.databases.has_key(db_name):
438 db = self.databases[db_name]
439 else:
440 db = self.databases[db_name] = self._opendb(db_name, 'c')
442 # now insert the journal entry
443 if db.has_key(nodeid):
444 # append to existing
445 s = db[nodeid]
446 l = marshal.loads(s)
447 l.append(entry)
448 else:
449 l = [entry]
451 db[nodeid] = marshal.dumps(l)
453 def _doStoreFile(self, name, **databases):
454 # the file is currently ".tmp" - move it to its real name to commit
455 os.rename(name+".tmp", name)
457 def rollback(self):
458 ''' Reverse all actions from the current transaction.
459 '''
460 if hyperdb.DEBUG:
461 print 'rollback', (self, )
462 for method, args in self.transactions:
463 # delete temporary files
464 if method == self._doStoreFile:
465 if os.path.exists(args[0]+".tmp"):
466 os.remove(args[0]+".tmp")
467 self.cache = {}
468 self.dirtynodes = {}
469 self.newnodes = {}
470 self.transactions = []
472 #
473 #$Log: not supported by cvs2svn $
474 #Revision 1.32 2002/04/15 23:25:15 richard
475 #. node ids are now generated from a lockable store - no more race conditions
476 #
477 #We're using the portalocker code by Jonathan Feinberg that was contributed
478 #to the ASPN Python cookbook. This gives us locking across Unix and Windows.
479 #
480 #Revision 1.31 2002/04/03 05:54:31 richard
481 #Fixed serialisation problem by moving the serialisation step out of the
482 #hyperdb.Class (get, set) into the hyperdb.Database.
483 #
484 #Also fixed htmltemplate after the showid changes I made yesterday.
485 #
486 #Unit tests for all of the above written.
487 #
488 #Revision 1.30 2002/02/27 03:40:59 richard
489 #Ran it through pychecker, made fixes
490 #
491 #Revision 1.29 2002/02/25 14:34:31 grubert
492 # . use blobfiles in back_anydbm which is used in back_bsddb.
493 # change test_db as dirlist does not work for subdirectories.
494 # ATTENTION: blobfiles now creates subdirectories for files.
495 #
496 #Revision 1.28 2002/02/16 09:14:17 richard
497 # . #514854 ] History: "User" is always ticket creator
498 #
499 #Revision 1.27 2002/01/22 07:21:13 richard
500 #. fixed back_bsddb so it passed the journal tests
501 #
502 #... it didn't seem happy using the back_anydbm _open method, which is odd.
503 #Yet another occurrance of whichdb not being able to recognise older bsddb
504 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
505 #process.
506 #
507 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
508 #last_set_entry was referenced before assignment
509 #
510 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
511 #We need to keep the last 'set' entry in the journal to preserve
512 #information on 'activity' for nodes.
513 #
514 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
515 #You can now use the roundup-admin tool to pack the database
516 #
517 #Revision 1.23 2002/01/18 04:32:04 richard
518 #Rollback was breaking because a message hadn't actually been written to the file. Needs
519 #more investigation.
520 #
521 #Revision 1.22 2002/01/14 02:20:15 richard
522 # . changed all config accesses so they access either the instance or the
523 # config attriubute on the db. This means that all config is obtained from
524 # instance_config instead of the mish-mash of classes. This will make
525 # switching to a ConfigParser setup easier too, I hope.
526 #
527 #At a minimum, this makes migration a _little_ easier (a lot easier in the
528 #0.5.0 switch, I hope!)
529 #
530 #Revision 1.21 2002/01/02 02:31:38 richard
531 #Sorry for the huge checkin message - I was only intending to implement #496356
532 #but I found a number of places where things had been broken by transactions:
533 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
534 # for _all_ roundup-generated smtp messages to be sent to.
535 # . the transaction cache had broken the roundupdb.Class set() reactors
536 # . newly-created author users in the mailgw weren't being committed to the db
537 #
538 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
539 #on when I found that stuff :):
540 # . #496356 ] Use threading in messages
541 # . detectors were being registered multiple times
542 # . added tests for mailgw
543 # . much better attaching of erroneous messages in the mail gateway
544 #
545 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
546 #Fixed bugs:
547 # . Fixed file creation and retrieval in same transaction in anydbm
548 # backend
549 # . Cgi interface now renders new issue after issue creation
550 # . Could not set issue status to resolved through cgi interface
551 # . Mail gateway was changing status back to 'chatting' if status was
552 # omitted as an argument
553 #
554 #Revision 1.19 2001/12/17 03:52:48 richard
555 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
556 #storing more than one file per node - if a property name is supplied,
557 #the file is called designator.property.
558 #I decided not to migrate the existing files stored over to the new naming
559 #scheme - the FileClass just doesn't specify the property name.
560 #
561 #Revision 1.18 2001/12/16 10:53:38 richard
562 #take a copy of the node dict so that the subsequent set
563 #operation doesn't modify the oldvalues structure
564 #
565 #Revision 1.17 2001/12/14 23:42:57 richard
566 #yuck, a gdbm instance tests false :(
567 #I've left the debugging code in - it should be removed one day if we're ever
568 #_really_ anal about performace :)
569 #
570 #Revision 1.16 2001/12/12 03:23:14 richard
571 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
572 #incorrectly identifies a dbm file as a dbhash file on my system. This has
573 #been submitted to the python bug tracker as issue #491888:
574 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
575 #
576 #Revision 1.15 2001/12/12 02:30:51 richard
577 #I fixed the problems with people whose anydbm was using the dbm module at the
578 #backend. It turns out the dbm module modifies the file name to append ".db"
579 #and my check to determine if we're opening an existing or new db just
580 #tested os.path.exists() on the filename. Well, no longer! We now perform a
581 #much better check _and_ cope with the anydbm implementation module changing
582 #too!
583 #I also fixed the backends __init__ so only ImportError is squashed.
584 #
585 #Revision 1.14 2001/12/10 22:20:01 richard
586 #Enabled transaction support in the bsddb backend. It uses the anydbm code
587 #where possible, only replacing methods where the db is opened (it uses the
588 #btree opener specifically.)
589 #Also cleaned up some change note generation.
590 #Made the backends package work with pydoc too.
591 #
592 #Revision 1.13 2001/12/02 05:06:16 richard
593 #. We now use weakrefs in the Classes to keep the database reference, so
594 # the close() method on the database is no longer needed.
595 # I bumped the minimum python requirement up to 2.1 accordingly.
596 #. #487480 ] roundup-server
597 #. #487476 ] INSTALL.txt
598 #
599 #I also cleaned up the change message / post-edit stuff in the cgi client.
600 #There's now a clearly marked "TODO: append the change note" where I believe
601 #the change note should be added there. The "changes" list will obviously
602 #have to be modified to be a dict of the changes, or somesuch.
603 #
604 #More testing needed.
605 #
606 #Revision 1.12 2001/12/01 07:17:50 richard
607 #. We now have basic transaction support! Information is only written to
608 # the database when the commit() method is called. Only the anydbm
609 # backend is modified in this way - neither of the bsddb backends have been.
610 # The mail, admin and cgi interfaces all use commit (except the admin tool
611 # doesn't have a commit command, so interactive users can't commit...)
612 #. Fixed login/registration forwarding the user to the right page (or not,
613 # on a failure)
614 #
615 #Revision 1.11 2001/11/21 02:34:18 richard
616 #Added a target version field to the extended issue schema
617 #
618 #Revision 1.10 2001/10/09 23:58:10 richard
619 #Moved the data stringification up into the hyperdb.Class class' get, set
620 #and create methods. This means that the data is also stringified for the
621 #journal call, and removes duplication of code from the backends. The
622 #backend code now only sees strings.
623 #
624 #Revision 1.9 2001/10/09 07:25:59 richard
625 #Added the Password property type. See "pydoc roundup.password" for
626 #implementation details. Have updated some of the documentation too.
627 #
628 #Revision 1.8 2001/09/29 13:27:00 richard
629 #CGI interfaces now spit up a top-level index of all the instances they can
630 #serve.
631 #
632 #Revision 1.7 2001/08/12 06:32:36 richard
633 #using isinstance(blah, Foo) now instead of isFooType
634 #
635 #Revision 1.6 2001/08/07 00:24:42 richard
636 #stupid typo
637 #
638 #Revision 1.5 2001/08/07 00:15:51 richard
639 #Added the copyright/license notice to (nearly) all files at request of
640 #Bizar Software.
641 #
642 #Revision 1.4 2001/07/30 01:41:36 richard
643 #Makes schema changes mucho easier.
644 #
645 #Revision 1.3 2001/07/25 01:23:07 richard
646 #Added the Roundup spec to the new documentation directory.
647 #
648 #Revision 1.2 2001/07/23 08:20:44 richard
649 #Moved over to using marshal in the bsddb and anydbm backends.
650 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
651 # retired - mod hyperdb.Class.list() so it lists retired nodes)
652 #
653 #