5f30b1ad6cffb3d2511a2d17070c84cdce4a9dd1
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: back_anydbm.py,v 1.28 2002-02-16 09:14:17 richard Exp $
19 '''
20 This module defines a backend that saves the hyperdatabase in a database
21 chosen by anydbm. It is guaranteed to always be available in python
22 versions >2.1.1 (the dumbdbm fallback in 2.1.1 and earlier has several
23 serious bugs, and is not available)
24 '''
26 import whichdb, anydbm, os, marshal
27 from roundup import hyperdb, date, password
29 #
30 # Now the database
31 #
32 class Database(hyperdb.Database):
33 """A database for storing records containing flexible data types.
35 Transaction stuff TODO:
36 . check the timestamp of the class file and nuke the cache if it's
37 modified. Do some sort of conflict checking on the dirty stuff.
38 . perhaps detect write collisions (related to above)?
40 """
41 def __init__(self, config, journaltag=None):
42 """Open a hyperdatabase given a specifier to some storage.
44 The 'storagelocator' is obtained from config.DATABASE.
45 The meaning of 'storagelocator' depends on the particular
46 implementation of the hyperdatabase. It could be a file name,
47 a directory path, a socket descriptor for a connection to a
48 database over the network, etc.
50 The 'journaltag' is a token that will be attached to the journal
51 entries for any edits done on the database. If 'journaltag' is
52 None, the database is opened in read-only mode: the Class.create(),
53 Class.set(), and Class.retire() methods are disabled.
54 """
55 self.config, self.journaltag = config, journaltag
56 self.dir = config.DATABASE
57 self.classes = {}
58 self.cache = {} # cache of nodes loaded or created
59 self.dirtynodes = {} # keep track of the dirty nodes by class
60 self.newnodes = {} # keep track of the new nodes by class
61 self.transactions = []
63 def __repr__(self):
64 return '<back_anydbm instance at %x>'%id(self)
66 #
67 # Classes
68 #
69 def __getattr__(self, classname):
70 """A convenient way of calling self.getclass(classname)."""
71 if self.classes.has_key(classname):
72 if hyperdb.DEBUG:
73 print '__getattr__', (self, classname)
74 return self.classes[classname]
75 raise AttributeError, classname
77 def addclass(self, cl):
78 if hyperdb.DEBUG:
79 print 'addclass', (self, cl)
80 cn = cl.classname
81 if self.classes.has_key(cn):
82 raise ValueError, cn
83 self.classes[cn] = cl
85 def getclasses(self):
86 """Return a list of the names of all existing classes."""
87 if hyperdb.DEBUG:
88 print 'getclasses', (self,)
89 l = self.classes.keys()
90 l.sort()
91 return l
93 def getclass(self, classname):
94 """Get the Class object representing a particular class.
96 If 'classname' is not a valid class name, a KeyError is raised.
97 """
98 if hyperdb.DEBUG:
99 print 'getclass', (self, classname)
100 return self.classes[classname]
102 #
103 # Class DBs
104 #
105 def clear(self):
106 '''Delete all database contents
107 '''
108 if hyperdb.DEBUG:
109 print 'clear', (self,)
110 for cn in self.classes.keys():
111 for type in 'nodes', 'journals':
112 path = os.path.join(self.dir, 'journals.%s'%cn)
113 if os.path.exists(path):
114 os.remove(path)
115 elif os.path.exists(path+'.db'): # dbm appends .db
116 os.remove(path+'.db')
118 def getclassdb(self, classname, mode='r'):
119 ''' grab a connection to the class db that will be used for
120 multiple actions
121 '''
122 if hyperdb.DEBUG:
123 print 'getclassdb', (self, classname, mode)
124 return self._opendb('nodes.%s'%classname, mode)
126 def _opendb(self, name, mode):
127 '''Low-level database opener that gets around anydbm/dbm
128 eccentricities.
129 '''
130 if hyperdb.DEBUG:
131 print '_opendb', (self, name, mode)
132 # determine which DB wrote the class file
133 db_type = ''
134 path = os.path.join(os.getcwd(), self.dir, name)
135 if os.path.exists(path):
136 db_type = whichdb.whichdb(path)
137 if not db_type:
138 raise hyperdb.DatabaseError, "Couldn't identify database type"
139 elif os.path.exists(path+'.db'):
140 # if the path ends in '.db', it's a dbm database, whether
141 # anydbm says it's dbhash or not!
142 db_type = 'dbm'
144 # new database? let anydbm pick the best dbm
145 if not db_type:
146 if hyperdb.DEBUG:
147 print "_opendb anydbm.open(%r, 'n')"%path
148 return anydbm.open(path, 'n')
150 # open the database with the correct module
151 try:
152 dbm = __import__(db_type)
153 except ImportError:
154 raise hyperdb.DatabaseError, \
155 "Couldn't open database - the required module '%s'"\
156 "is not available"%db_type
157 if hyperdb.DEBUG:
158 print "_opendb %r.open(%r, %r)"%(db_type, path, mode)
159 return dbm.open(path, mode)
161 #
162 # Nodes
163 #
164 def addnode(self, classname, nodeid, node):
165 ''' add the specified node to its class's db
166 '''
167 if hyperdb.DEBUG:
168 print 'addnode', (self, classname, nodeid, node)
169 self.newnodes.setdefault(classname, {})[nodeid] = 1
170 self.cache.setdefault(classname, {})[nodeid] = node
171 self.savenode(classname, nodeid, node)
173 def setnode(self, classname, nodeid, node):
174 ''' change the specified node
175 '''
176 if hyperdb.DEBUG:
177 print 'setnode', (self, classname, nodeid, node)
178 self.dirtynodes.setdefault(classname, {})[nodeid] = 1
179 # can't set without having already loaded the node
180 self.cache[classname][nodeid] = node
181 self.savenode(classname, nodeid, node)
183 def savenode(self, classname, nodeid, node):
184 ''' perform the saving of data specified by the set/addnode
185 '''
186 if hyperdb.DEBUG:
187 print 'savenode', (self, classname, nodeid, node)
188 self.transactions.append((self._doSaveNode, (classname, nodeid, node)))
190 def getnode(self, classname, nodeid, db=None, cache=1):
191 ''' get a node from the database
192 '''
193 if hyperdb.DEBUG:
194 print 'getnode', (self, classname, nodeid, db)
195 if cache:
196 # try the cache
197 cache = self.cache.setdefault(classname, {})
198 if cache.has_key(nodeid):
199 return cache[nodeid]
201 # get from the database and save in the cache
202 if db is None:
203 db = self.getclassdb(classname)
204 if not db.has_key(nodeid):
205 raise IndexError, "no such %s %s"%(classname, nodeid)
206 res = marshal.loads(db[nodeid])
207 if cache:
208 cache[nodeid] = res
209 return res
211 def hasnode(self, classname, nodeid, db=None):
212 ''' determine if the database has a given node
213 '''
214 if hyperdb.DEBUG:
215 print 'hasnode', (self, classname, nodeid, db)
216 # try the cache
217 cache = self.cache.setdefault(classname, {})
218 if cache.has_key(nodeid):
219 return 1
221 # not in the cache - check the database
222 if db is None:
223 db = self.getclassdb(classname)
224 res = db.has_key(nodeid)
225 return res
227 def countnodes(self, classname, db=None):
228 if hyperdb.DEBUG:
229 print 'countnodes', (self, classname, db)
230 # include the new nodes not saved to the DB yet
231 count = len(self.newnodes.get(classname, {}))
233 # and count those in the DB
234 if db is None:
235 db = self.getclassdb(classname)
236 count = count + len(db.keys())
237 return count
239 def getnodeids(self, classname, db=None):
240 if hyperdb.DEBUG:
241 print 'getnodeids', (self, classname, db)
242 # start off with the new nodes
243 res = self.newnodes.get(classname, {}).keys()
245 if db is None:
246 db = self.getclassdb(classname)
247 res = res + db.keys()
248 return res
251 #
252 # Files - special node properties
253 #
254 def filename(self, classname, nodeid, property=None):
255 '''Determine what the filename for the given node and optionally property is.
256 '''
257 # TODO: split into multiple files directories
258 if property:
259 return os.path.join(self.dir, 'files', '%s%s.%s'%(classname,
260 nodeid, property))
261 else:
262 # roundupdb.FileClass never specified the property name, so don't include it
263 return os.path.join(self.dir, 'files', '%s%s'%(classname,
264 nodeid))
266 def storefile(self, classname, nodeid, property, content):
267 '''Store the content of the file in the database. The property may be None, in
268 which case the filename does not indicate which property is being saved.
269 '''
270 name = self.filename(classname, nodeid, property)
271 open(name + '.tmp', 'wb').write(content)
272 self.transactions.append((self._doStoreFile, (name, )))
274 def getfile(self, classname, nodeid, property):
275 '''Store the content of the file in the database.
276 '''
277 filename = self.filename(classname, nodeid, property)
278 try:
279 return open(filename, 'rb').read()
280 except:
281 return open(filename+'.tmp', 'rb').read()
284 #
285 # Journal
286 #
287 def addjournal(self, classname, nodeid, action, params):
288 ''' Journal the Action
289 'action' may be:
291 'create' or 'set' -- 'params' is a dictionary of property values
292 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
293 'retire' -- 'params' is None
294 '''
295 if hyperdb.DEBUG:
296 print 'addjournal', (self, classname, nodeid, action, params)
297 self.transactions.append((self._doSaveJournal, (classname, nodeid,
298 action, params)))
300 def getjournal(self, classname, nodeid):
301 ''' get the journal for id
302 '''
303 if hyperdb.DEBUG:
304 print 'getjournal', (self, classname, nodeid)
305 # attempt to open the journal - in some rare cases, the journal may
306 # not exist
307 try:
308 db = self._opendb('journals.%s'%classname, 'r')
309 except anydbm.error, error:
310 if str(error) == "need 'c' or 'n' flag to open new db": return []
311 elif error.args[0] != 2: raise
312 return []
313 journal = marshal.loads(db[nodeid])
314 res = []
315 for entry in journal:
316 (nodeid, date_stamp, user, action, params) = entry
317 date_obj = date.Date(date_stamp)
318 res.append((nodeid, date_obj, user, action, params))
319 return res
321 def pack(self, pack_before):
322 ''' delete all journal entries before 'pack_before' '''
323 if hyperdb.DEBUG:
324 print 'packjournal', (self, pack_before)
326 pack_before = pack_before.get_tuple()
328 classes = self.getclasses()
330 # TODO: factor this out to method - we're already doing it in
331 # _opendb.
332 db_type = ''
333 path = os.path.join(os.getcwd(), self.dir, classes[0])
334 if os.path.exists(path):
335 db_type = whichdb.whichdb(path)
336 if not db_type:
337 raise hyperdb.DatabaseError, "Couldn't identify database type"
338 elif os.path.exists(path+'.db'):
339 db_type = 'dbm'
341 for classname in classes:
342 db_name = 'journals.%s'%classname
343 db = self._opendb(db_name, 'w')
345 for key in db.keys():
346 journal = marshal.loads(db[key])
347 l = []
348 last_set_entry = None
349 for entry in journal:
350 (nodeid, date_stamp, self.journaltag, action,
351 params) = entry
352 if date_stamp > pack_before or action == 'create':
353 l.append(entry)
354 elif action == 'set':
355 # grab the last set entry to keep information on
356 # activity
357 last_set_entry = entry
358 if last_set_entry:
359 date_stamp = last_set_entry[1]
360 # if the last set entry was made after the pack date
361 # then it is already in the list
362 if date_stamp < pack_before:
363 l.append(last_set_entry)
364 db[key] = marshal.dumps(l)
365 if db_type == 'gdbm':
366 db.reorganize()
367 db.close()
370 #
371 # Basic transaction support
372 #
373 def commit(self):
374 ''' Commit the current transactions.
375 '''
376 if hyperdb.DEBUG:
377 print 'commit', (self,)
378 # TODO: lock the DB
380 # keep a handle to all the database files opened
381 self.databases = {}
383 # now, do all the transactions
384 for method, args in self.transactions:
385 method(*args)
387 # now close all the database files
388 for db in self.databases.values():
389 db.close()
390 del self.databases
391 # TODO: unlock the DB
393 # all transactions committed, back to normal
394 self.cache = {}
395 self.dirtynodes = {}
396 self.newnodes = {}
397 self.transactions = []
399 def _doSaveNode(self, classname, nodeid, node):
400 if hyperdb.DEBUG:
401 print '_doSaveNode', (self, classname, nodeid, node)
403 # get the database handle
404 db_name = 'nodes.%s'%classname
405 if self.databases.has_key(db_name):
406 db = self.databases[db_name]
407 else:
408 db = self.databases[db_name] = self.getclassdb(classname, 'c')
410 # now save the marshalled data
411 db[nodeid] = marshal.dumps(node)
413 def _doSaveJournal(self, classname, nodeid, action, params):
414 entry = (nodeid, date.Date().get_tuple(), self.journaltag, action,
415 params)
416 if hyperdb.DEBUG:
417 print '_doSaveJournal', entry
419 # get the database handle
420 db_name = 'journals.%s'%classname
421 if self.databases.has_key(db_name):
422 db = self.databases[db_name]
423 else:
424 db = self.databases[db_name] = self._opendb(db_name, 'c')
426 # now insert the journal entry
427 if db.has_key(nodeid):
428 s = db[nodeid]
429 l = marshal.loads(db[nodeid])
430 l.append(entry)
431 else:
432 l = [entry]
433 db[nodeid] = marshal.dumps(l)
435 def _doStoreFile(self, name, **databases):
436 # the file is currently ".tmp" - move it to its real name to commit
437 os.rename(name+".tmp", name)
439 def rollback(self):
440 ''' Reverse all actions from the current transaction.
441 '''
442 if hyperdb.DEBUG:
443 print 'rollback', (self, )
444 for method, args in self.transactions:
445 # delete temporary files
446 if method == self._doStoreFile:
447 if os.path.exists(args[0]+".tmp"):
448 os.remove(args[0]+".tmp")
449 self.cache = {}
450 self.dirtynodes = {}
451 self.newnodes = {}
452 self.transactions = []
454 #
455 #$Log: not supported by cvs2svn $
456 #Revision 1.27 2002/01/22 07:21:13 richard
457 #. fixed back_bsddb so it passed the journal tests
458 #
459 #... it didn't seem happy using the back_anydbm _open method, which is odd.
460 #Yet another occurrance of whichdb not being able to recognise older bsddb
461 #databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
462 #process.
463 #
464 #Revision 1.26 2002/01/22 05:18:38 rochecompaan
465 #last_set_entry was referenced before assignment
466 #
467 #Revision 1.25 2002/01/22 05:06:08 rochecompaan
468 #We need to keep the last 'set' entry in the journal to preserve
469 #information on 'activity' for nodes.
470 #
471 #Revision 1.24 2002/01/21 16:33:20 rochecompaan
472 #You can now use the roundup-admin tool to pack the database
473 #
474 #Revision 1.23 2002/01/18 04:32:04 richard
475 #Rollback was breaking because a message hadn't actually been written to the file. Needs
476 #more investigation.
477 #
478 #Revision 1.22 2002/01/14 02:20:15 richard
479 # . changed all config accesses so they access either the instance or the
480 # config attriubute on the db. This means that all config is obtained from
481 # instance_config instead of the mish-mash of classes. This will make
482 # switching to a ConfigParser setup easier too, I hope.
483 #
484 #At a minimum, this makes migration a _little_ easier (a lot easier in the
485 #0.5.0 switch, I hope!)
486 #
487 #Revision 1.21 2002/01/02 02:31:38 richard
488 #Sorry for the huge checkin message - I was only intending to implement #496356
489 #but I found a number of places where things had been broken by transactions:
490 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
491 # for _all_ roundup-generated smtp messages to be sent to.
492 # . the transaction cache had broken the roundupdb.Class set() reactors
493 # . newly-created author users in the mailgw weren't being committed to the db
494 #
495 #Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
496 #on when I found that stuff :):
497 # . #496356 ] Use threading in messages
498 # . detectors were being registered multiple times
499 # . added tests for mailgw
500 # . much better attaching of erroneous messages in the mail gateway
501 #
502 #Revision 1.20 2001/12/18 15:30:34 rochecompaan
503 #Fixed bugs:
504 # . Fixed file creation and retrieval in same transaction in anydbm
505 # backend
506 # . Cgi interface now renders new issue after issue creation
507 # . Could not set issue status to resolved through cgi interface
508 # . Mail gateway was changing status back to 'chatting' if status was
509 # omitted as an argument
510 #
511 #Revision 1.19 2001/12/17 03:52:48 richard
512 #Implemented file store rollback. As a bonus, the hyperdb is now capable of
513 #storing more than one file per node - if a property name is supplied,
514 #the file is called designator.property.
515 #I decided not to migrate the existing files stored over to the new naming
516 #scheme - the FileClass just doesn't specify the property name.
517 #
518 #Revision 1.18 2001/12/16 10:53:38 richard
519 #take a copy of the node dict so that the subsequent set
520 #operation doesn't modify the oldvalues structure
521 #
522 #Revision 1.17 2001/12/14 23:42:57 richard
523 #yuck, a gdbm instance tests false :(
524 #I've left the debugging code in - it should be removed one day if we're ever
525 #_really_ anal about performace :)
526 #
527 #Revision 1.16 2001/12/12 03:23:14 richard
528 #Cor blimey this anydbm/whichdb stuff is yecchy. Turns out that whichdb
529 #incorrectly identifies a dbm file as a dbhash file on my system. This has
530 #been submitted to the python bug tracker as issue #491888:
531 #https://sourceforge.net/tracker/index.php?func=detail&aid=491888&group_id=5470&atid=105470
532 #
533 #Revision 1.15 2001/12/12 02:30:51 richard
534 #I fixed the problems with people whose anydbm was using the dbm module at the
535 #backend. It turns out the dbm module modifies the file name to append ".db"
536 #and my check to determine if we're opening an existing or new db just
537 #tested os.path.exists() on the filename. Well, no longer! We now perform a
538 #much better check _and_ cope with the anydbm implementation module changing
539 #too!
540 #I also fixed the backends __init__ so only ImportError is squashed.
541 #
542 #Revision 1.14 2001/12/10 22:20:01 richard
543 #Enabled transaction support in the bsddb backend. It uses the anydbm code
544 #where possible, only replacing methods where the db is opened (it uses the
545 #btree opener specifically.)
546 #Also cleaned up some change note generation.
547 #Made the backends package work with pydoc too.
548 #
549 #Revision 1.13 2001/12/02 05:06:16 richard
550 #. We now use weakrefs in the Classes to keep the database reference, so
551 # the close() method on the database is no longer needed.
552 # I bumped the minimum python requirement up to 2.1 accordingly.
553 #. #487480 ] roundup-server
554 #. #487476 ] INSTALL.txt
555 #
556 #I also cleaned up the change message / post-edit stuff in the cgi client.
557 #There's now a clearly marked "TODO: append the change note" where I believe
558 #the change note should be added there. The "changes" list will obviously
559 #have to be modified to be a dict of the changes, or somesuch.
560 #
561 #More testing needed.
562 #
563 #Revision 1.12 2001/12/01 07:17:50 richard
564 #. We now have basic transaction support! Information is only written to
565 # the database when the commit() method is called. Only the anydbm
566 # backend is modified in this way - neither of the bsddb backends have been.
567 # The mail, admin and cgi interfaces all use commit (except the admin tool
568 # doesn't have a commit command, so interactive users can't commit...)
569 #. Fixed login/registration forwarding the user to the right page (or not,
570 # on a failure)
571 #
572 #Revision 1.11 2001/11/21 02:34:18 richard
573 #Added a target version field to the extended issue schema
574 #
575 #Revision 1.10 2001/10/09 23:58:10 richard
576 #Moved the data stringification up into the hyperdb.Class class' get, set
577 #and create methods. This means that the data is also stringified for the
578 #journal call, and removes duplication of code from the backends. The
579 #backend code now only sees strings.
580 #
581 #Revision 1.9 2001/10/09 07:25:59 richard
582 #Added the Password property type. See "pydoc roundup.password" for
583 #implementation details. Have updated some of the documentation too.
584 #
585 #Revision 1.8 2001/09/29 13:27:00 richard
586 #CGI interfaces now spit up a top-level index of all the instances they can
587 #serve.
588 #
589 #Revision 1.7 2001/08/12 06:32:36 richard
590 #using isinstance(blah, Foo) now instead of isFooType
591 #
592 #Revision 1.6 2001/08/07 00:24:42 richard
593 #stupid typo
594 #
595 #Revision 1.5 2001/08/07 00:15:51 richard
596 #Added the copyright/license notice to (nearly) all files at request of
597 #Bizar Software.
598 #
599 #Revision 1.4 2001/07/30 01:41:36 richard
600 #Makes schema changes mucho easier.
601 #
602 #Revision 1.3 2001/07/25 01:23:07 richard
603 #Added the Roundup spec to the new documentation directory.
604 #
605 #Revision 1.2 2001/07/23 08:20:44 richard
606 #Moved over to using marshal in the bsddb and anydbm backends.
607 #roundup-admin now has a "freshen" command that'll load/save all nodes (not
608 # retired - mod hyperdb.Class.list() so it lists retired nodes)
609 #
610 #