1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 # $Id: hyperdb.py,v 1.59 2002-03-12 22:52:26 richard Exp $
20 __doc__ = """
21 Hyperdatabase implementation, especially field types.
22 """
24 # standard python modules
25 import re, string, weakref, os
27 # roundup modules
28 import date, password
30 DEBUG = os.environ.get('HYPERDBDEBUG', '')
32 #
33 # Types
34 #
35 class String:
36 """An object designating a String property."""
37 def __repr__(self):
38 ' more useful for dumps '
39 return '<%s>'%self.__class__
41 class Password:
42 """An object designating a Password property."""
43 def __repr__(self):
44 ' more useful for dumps '
45 return '<%s>'%self.__class__
47 class Date:
48 """An object designating a Date property."""
49 def __repr__(self):
50 ' more useful for dumps '
51 return '<%s>'%self.__class__
53 class Interval:
54 """An object designating an Interval property."""
55 def __repr__(self):
56 ' more useful for dumps '
57 return '<%s>'%self.__class__
59 class Link:
60 """An object designating a Link property that links to a
61 node in a specified class."""
62 def __init__(self, classname, do_journal='no'):
63 ''' Default is to not journal link and unlink events
64 '''
65 self.classname = classname
66 self.do_journal = do_journal == 'yes'
67 def __repr__(self):
68 ' more useful for dumps '
69 return '<%s to "%s">'%(self.__class__, self.classname)
71 class Multilink:
72 """An object designating a Multilink property that links
73 to nodes in a specified class.
75 "classname" indicates the class to link to
77 "do_journal" indicates whether the linked-to nodes should have
78 'link' and 'unlink' events placed in their journal
79 """
80 def __init__(self, classname, do_journal='no'):
81 ''' Default is to not journal link and unlink events
82 '''
83 self.classname = classname
84 self.do_journal = do_journal == 'yes'
85 def __repr__(self):
86 ' more useful for dumps '
87 return '<%s to "%s">'%(self.__class__, self.classname)
89 class DatabaseError(ValueError):
90 '''Error to be raised when there is some problem in the database code
91 '''
92 pass
95 #
96 # the base Database class
97 #
98 class Database:
99 '''A database for storing records containing flexible data types.
101 This class defines a hyperdatabase storage layer, which the Classes use to
102 store their data.
105 Transactions
106 ------------
107 The Database should support transactions through the commit() and
108 rollback() methods. All other Database methods should be transaction-aware,
109 using data from the current transaction before looking up the database.
111 An implementation must provide an override for the get() method so that the
112 in-database value is returned in preference to the in-transaction value.
113 This is necessary to determine if any values have changed during a
114 transaction.
116 '''
118 # flag to set on retired entries
119 RETIRED_FLAG = '__hyperdb_retired'
121 # XXX deviates from spec: storagelocator is obtained from the config
122 def __init__(self, config, journaltag=None):
123 """Open a hyperdatabase given a specifier to some storage.
125 The 'storagelocator' is obtained from config.DATABASE.
126 The meaning of 'storagelocator' depends on the particular
127 implementation of the hyperdatabase. It could be a file name,
128 a directory path, a socket descriptor for a connection to a
129 database over the network, etc.
131 The 'journaltag' is a token that will be attached to the journal
132 entries for any edits done on the database. If 'journaltag' is
133 None, the database is opened in read-only mode: the Class.create(),
134 Class.set(), and Class.retire() methods are disabled.
135 """
136 raise NotImplementedError
138 def __getattr__(self, classname):
139 """A convenient way of calling self.getclass(classname)."""
140 raise NotImplementedError
142 def addclass(self, cl):
143 '''Add a Class to the hyperdatabase.
144 '''
145 raise NotImplementedError
147 def getclasses(self):
148 """Return a list of the names of all existing classes."""
149 raise NotImplementedError
151 def getclass(self, classname):
152 """Get the Class object representing a particular class.
154 If 'classname' is not a valid class name, a KeyError is raised.
155 """
156 raise NotImplementedError
158 def clear(self):
159 '''Delete all database contents.
160 '''
161 raise NotImplementedError
163 def getclassdb(self, classname, mode='r'):
164 '''Obtain a connection to the class db that will be used for
165 multiple actions.
166 '''
167 raise NotImplementedError
169 def addnode(self, classname, nodeid, node):
170 '''Add the specified node to its class's db.
171 '''
172 raise NotImplementedError
174 def setnode(self, classname, nodeid, node):
175 '''Change the specified node.
176 '''
177 raise NotImplementedError
179 def getnode(self, classname, nodeid, db=None, cache=1):
180 '''Get a node from the database.
181 '''
182 raise NotImplementedError
184 def hasnode(self, classname, nodeid, db=None):
185 '''Determine if the database has a given node.
186 '''
187 raise NotImplementedError
189 def countnodes(self, classname, db=None):
190 '''Count the number of nodes that exist for a particular Class.
191 '''
192 raise NotImplementedError
194 def getnodeids(self, classname, db=None):
195 '''Retrieve all the ids of the nodes for a particular Class.
196 '''
197 raise NotImplementedError
199 def storefile(self, classname, nodeid, property, content):
200 '''Store the content of the file in the database.
202 The property may be None, in which case the filename does not
203 indicate which property is being saved.
204 '''
205 raise NotImplementedError
207 def getfile(self, classname, nodeid, property):
208 '''Store the content of the file in the database.
209 '''
210 raise NotImplementedError
212 def addjournal(self, classname, nodeid, action, params):
213 ''' Journal the Action
214 'action' may be:
216 'create' or 'set' -- 'params' is a dictionary of property values
217 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
218 'retire' -- 'params' is None
219 '''
220 raise NotImplementedError
222 def getjournal(self, classname, nodeid):
223 ''' get the journal for id
224 '''
225 raise NotImplementedError
227 def pack(self, pack_before):
228 ''' pack the database
229 '''
230 raise NotImplementedError
232 def commit(self):
233 ''' Commit the current transactions.
235 Save all data changed since the database was opened or since the
236 last commit() or rollback().
237 '''
238 raise NotImplementedError
240 def rollback(self):
241 ''' Reverse all actions from the current transaction.
243 Undo all the changes made since the database was opened or the last
244 commit() or rollback() was performed.
245 '''
246 raise NotImplementedError
248 _marker = []
249 #
250 # The base Class class
251 #
252 class Class:
253 """The handle to a particular class of nodes in a hyperdatabase."""
255 def __init__(self, db, classname, **properties):
256 """Create a new class with a given name and property specification.
258 'classname' must not collide with the name of an existing class,
259 or a ValueError is raised. The keyword arguments in 'properties'
260 must map names to property objects, or a TypeError is raised.
261 """
262 self.classname = classname
263 self.properties = properties
264 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
265 self.key = ''
267 # do the db-related init stuff
268 db.addclass(self)
270 def __repr__(self):
271 '''Slightly more useful representation
272 '''
273 return '<hypderdb.Class "%s">'%self.classname
275 # Editing nodes:
277 def create(self, **propvalues):
278 """Create a new node of this class and return its id.
280 The keyword arguments in 'propvalues' map property names to values.
282 The values of arguments must be acceptable for the types of their
283 corresponding properties or a TypeError is raised.
285 If this class has a key property, it must be present and its value
286 must not collide with other key strings or a ValueError is raised.
288 Any other properties on this class that are missing from the
289 'propvalues' dictionary are set to None.
291 If an id in a link or multilink property does not refer to a valid
292 node, an IndexError is raised.
293 """
294 if propvalues.has_key('id'):
295 raise KeyError, '"id" is reserved'
297 if self.db.journaltag is None:
298 raise DatabaseError, 'Database open read-only'
300 # new node's id
301 newid = str(self.count() + 1)
303 # validate propvalues
304 num_re = re.compile('^\d+$')
305 for key, value in propvalues.items():
306 if key == self.key:
307 try:
308 self.lookup(value)
309 except KeyError:
310 pass
311 else:
312 raise ValueError, 'node with key "%s" exists'%value
314 # try to handle this property
315 try:
316 prop = self.properties[key]
317 except KeyError:
318 raise KeyError, '"%s" has no property "%s"'%(self.classname,
319 key)
321 if isinstance(prop, Link):
322 if type(value) != type(''):
323 raise ValueError, 'link value must be String'
324 link_class = self.properties[key].classname
325 # if it isn't a number, it's a key
326 if not num_re.match(value):
327 try:
328 value = self.db.classes[link_class].lookup(value)
329 except (TypeError, KeyError):
330 raise IndexError, 'new property "%s": %s not a %s'%(
331 key, value, link_class)
332 elif not self.db.hasnode(link_class, value):
333 raise IndexError, '%s has no node %s'%(link_class, value)
335 # save off the value
336 propvalues[key] = value
338 # register the link with the newly linked node
339 if self.properties[key].do_journal:
340 self.db.addjournal(link_class, value, 'link',
341 (self.classname, newid, key))
343 elif isinstance(prop, Multilink):
344 if type(value) != type([]):
345 raise TypeError, 'new property "%s" not a list of ids'%key
346 link_class = self.properties[key].classname
347 l = []
348 for entry in value:
349 if type(entry) != type(''):
350 raise ValueError, 'link value must be String'
351 # if it isn't a number, it's a key
352 if not num_re.match(entry):
353 try:
354 entry = self.db.classes[link_class].lookup(entry)
355 except (TypeError, KeyError):
356 raise IndexError, 'new property "%s": %s not a %s'%(
357 key, entry, self.properties[key].classname)
358 l.append(entry)
359 value = l
360 propvalues[key] = value
362 # handle additions
363 for id in value:
364 if not self.db.hasnode(link_class, id):
365 raise IndexError, '%s has no node %s'%(link_class, id)
366 # register the link with the newly linked node
367 if self.properties[key].do_journal:
368 self.db.addjournal(link_class, id, 'link',
369 (self.classname, newid, key))
371 elif isinstance(prop, String):
372 if type(value) != type(''):
373 raise TypeError, 'new property "%s" not a string'%key
375 elif isinstance(prop, Password):
376 if not isinstance(value, password.Password):
377 raise TypeError, 'new property "%s" not a Password'%key
379 elif isinstance(prop, Date):
380 if value is not None and not isinstance(value, date.Date):
381 raise TypeError, 'new property "%s" not a Date'%key
383 elif isinstance(prop, Interval):
384 if value is not None and not isinstance(value, date.Interval):
385 raise TypeError, 'new property "%s" not an Interval'%key
387 # make sure there's data where there needs to be
388 for key, prop in self.properties.items():
389 if propvalues.has_key(key):
390 continue
391 if key == self.key:
392 raise ValueError, 'key property "%s" is required'%key
393 if isinstance(prop, Multilink):
394 propvalues[key] = []
395 else:
396 # TODO: None isn't right here, I think...
397 propvalues[key] = None
399 # convert all data to strings
400 for key, prop in self.properties.items():
401 if isinstance(prop, Date):
402 if propvalues[key] is not None:
403 propvalues[key] = propvalues[key].get_tuple()
404 elif isinstance(prop, Interval):
405 if propvalues[key] is not None:
406 propvalues[key] = propvalues[key].get_tuple()
407 elif isinstance(prop, Password):
408 propvalues[key] = str(propvalues[key])
410 # done
411 self.db.addnode(self.classname, newid, propvalues)
412 self.db.addjournal(self.classname, newid, 'create', propvalues)
413 return newid
415 def get(self, nodeid, propname, default=_marker, cache=1):
416 """Get the value of a property on an existing node of this class.
418 'nodeid' must be the id of an existing node of this class or an
419 IndexError is raised. 'propname' must be the name of a property
420 of this class or a KeyError is raised.
422 'cache' indicates whether the transaction cache should be queried
423 for the node. If the node has been modified and you need to
424 determine what its values prior to modification are, you need to
425 set cache=0.
426 """
427 if propname == 'id':
428 return nodeid
430 # get the property (raises KeyErorr if invalid)
431 prop = self.properties[propname]
433 # get the node's dict
434 d = self.db.getnode(self.classname, nodeid, cache=cache)
436 if not d.has_key(propname):
437 if default is _marker:
438 if isinstance(prop, Multilink):
439 return []
440 else:
441 # TODO: None isn't right here, I think...
442 return None
443 else:
444 return default
446 # possibly convert the marshalled data to instances
447 if isinstance(prop, Date):
448 if d[propname] is None:
449 return None
450 return date.Date(d[propname])
451 elif isinstance(prop, Interval):
452 if d[propname] is None:
453 return None
454 return date.Interval(d[propname])
455 elif isinstance(prop, Password):
456 p = password.Password()
457 p.unpack(d[propname])
458 return p
460 return d[propname]
462 # XXX not in spec
463 def getnode(self, nodeid, cache=1):
464 ''' Return a convenience wrapper for the node.
466 'nodeid' must be the id of an existing node of this class or an
467 IndexError is raised.
469 'cache' indicates whether the transaction cache should be queried
470 for the node. If the node has been modified and you need to
471 determine what its values prior to modification are, you need to
472 set cache=0.
473 '''
474 return Node(self, nodeid, cache=cache)
476 def set(self, nodeid, **propvalues):
477 """Modify a property on an existing node of this class.
479 'nodeid' must be the id of an existing node of this class or an
480 IndexError is raised.
482 Each key in 'propvalues' must be the name of a property of this
483 class or a KeyError is raised.
485 All values in 'propvalues' must be acceptable types for their
486 corresponding properties or a TypeError is raised.
488 If the value of the key property is set, it must not collide with
489 other key strings or a ValueError is raised.
491 If the value of a Link or Multilink property contains an invalid
492 node id, a ValueError is raised.
493 """
494 if not propvalues:
495 return
497 if propvalues.has_key('id'):
498 raise KeyError, '"id" is reserved'
500 if self.db.journaltag is None:
501 raise DatabaseError, 'Database open read-only'
503 node = self.db.getnode(self.classname, nodeid)
504 if node.has_key(self.db.RETIRED_FLAG):
505 raise IndexError
506 num_re = re.compile('^\d+$')
507 for key, value in propvalues.items():
508 # check to make sure we're not duplicating an existing key
509 if key == self.key and node[key] != value:
510 try:
511 self.lookup(value)
512 except KeyError:
513 pass
514 else:
515 raise ValueError, 'node with key "%s" exists'%value
517 # this will raise the KeyError if the property isn't valid
518 # ... we don't use getprops() here because we only care about
519 # the writeable properties.
520 prop = self.properties[key]
522 # if the value's the same as the existing value, no sense in
523 # doing anything
524 if node.has_key(key) and value == node[key]:
525 del propvalues[key]
526 continue
528 # do stuff based on the prop type
529 if isinstance(prop, Link):
530 link_class = self.properties[key].classname
531 # if it isn't a number, it's a key
532 if type(value) != type(''):
533 raise ValueError, 'link value must be String'
534 if not num_re.match(value):
535 try:
536 value = self.db.classes[link_class].lookup(value)
537 except (TypeError, KeyError):
538 raise IndexError, 'new property "%s": %s not a %s'%(
539 key, value, self.properties[key].classname)
541 if not self.db.hasnode(link_class, value):
542 raise IndexError, '%s has no node %s'%(link_class, value)
544 if self.properties[key].do_journal:
545 # register the unlink with the old linked node
546 if node[key] is not None:
547 self.db.addjournal(link_class, node[key], 'unlink',
548 (self.classname, nodeid, key))
550 # register the link with the newly linked node
551 if value is not None:
552 self.db.addjournal(link_class, value, 'link',
553 (self.classname, nodeid, key))
555 elif isinstance(prop, Multilink):
556 if type(value) != type([]):
557 raise TypeError, 'new property "%s" not a list of ids'%key
558 link_class = self.properties[key].classname
559 l = []
560 for entry in value:
561 # if it isn't a number, it's a key
562 if type(entry) != type(''):
563 raise ValueError, 'link value must be String'
564 if not num_re.match(entry):
565 try:
566 entry = self.db.classes[link_class].lookup(entry)
567 except (TypeError, KeyError):
568 raise IndexError, 'new property "%s": %s not a %s'%(
569 key, entry, self.properties[key].classname)
570 l.append(entry)
571 value = l
572 propvalues[key] = value
574 # handle removals
575 if node.has_key(key):
576 l = node[key]
577 else:
578 l = []
579 for id in l[:]:
580 if id in value:
581 continue
582 # register the unlink with the old linked node
583 if self.properties[key].do_journal:
584 self.db.addjournal(link_class, id, 'unlink',
585 (self.classname, nodeid, key))
586 l.remove(id)
588 # handle additions
589 for id in value:
590 if not self.db.hasnode(link_class, id):
591 raise IndexError, '%s has no node %s'%(
592 link_class, id)
593 if id in l:
594 continue
595 # register the link with the newly linked node
596 if self.properties[key].do_journal:
597 self.db.addjournal(link_class, id, 'link',
598 (self.classname, nodeid, key))
599 l.append(id)
601 elif isinstance(prop, String):
602 if value is not None and type(value) != type(''):
603 raise TypeError, 'new property "%s" not a string'%key
605 elif isinstance(prop, Password):
606 if not isinstance(value, password.Password):
607 raise TypeError, 'new property "%s" not a Password'% key
608 propvalues[key] = value = str(value)
610 elif value is not None and isinstance(prop, Date):
611 if not isinstance(value, date.Date):
612 raise TypeError, 'new property "%s" not a Date'% key
613 propvalues[key] = value = value.get_tuple()
615 elif value is not None and isinstance(prop, Interval):
616 if not isinstance(value, date.Interval):
617 raise TypeError, 'new property "%s" not an Interval'% key
618 propvalues[key] = value = value.get_tuple()
620 node[key] = value
622 # nothing to do?
623 if not propvalues:
624 return
626 # do the set, and journal it
627 self.db.setnode(self.classname, nodeid, node)
628 self.db.addjournal(self.classname, nodeid, 'set', propvalues)
630 def retire(self, nodeid):
631 """Retire a node.
633 The properties on the node remain available from the get() method,
634 and the node's id is never reused.
636 Retired nodes are not returned by the find(), list(), or lookup()
637 methods, and other nodes may reuse the values of their key properties.
638 """
639 if self.db.journaltag is None:
640 raise DatabaseError, 'Database open read-only'
641 node = self.db.getnode(self.classname, nodeid)
642 node[self.db.RETIRED_FLAG] = 1
643 self.db.setnode(self.classname, nodeid, node)
644 self.db.addjournal(self.classname, nodeid, 'retired', None)
646 def history(self, nodeid):
647 """Retrieve the journal of edits on a particular node.
649 'nodeid' must be the id of an existing node of this class or an
650 IndexError is raised.
652 The returned list contains tuples of the form
654 (date, tag, action, params)
656 'date' is a Timestamp object specifying the time of the change and
657 'tag' is the journaltag specified when the database was opened.
658 """
659 return self.db.getjournal(self.classname, nodeid)
661 # Locating nodes:
662 def hasnode(self, nodeid):
663 '''Determine if the given nodeid actually exists
664 '''
665 return self.db.hasnode(self.classname, nodeid)
667 def setkey(self, propname):
668 """Select a String property of this class to be the key property.
670 'propname' must be the name of a String property of this class or
671 None, or a TypeError is raised. The values of the key property on
672 all existing nodes must be unique or a ValueError is raised.
673 """
674 # TODO: validate that the property is a String!
675 self.key = propname
677 def getkey(self):
678 """Return the name of the key property for this class or None."""
679 return self.key
681 def labelprop(self, default_to_id=0):
682 ''' Return the property name for a label for the given node.
684 This method attempts to generate a consistent label for the node.
685 It tries the following in order:
686 1. key property
687 2. "name" property
688 3. "title" property
689 4. first property from the sorted property name list
690 '''
691 k = self.getkey()
692 if k:
693 return k
694 props = self.getprops()
695 if props.has_key('name'):
696 return 'name'
697 elif props.has_key('title'):
698 return 'title'
699 if default_to_id:
700 return 'id'
701 props = props.keys()
702 props.sort()
703 return props[0]
705 # TODO: set up a separate index db file for this? profile?
706 def lookup(self, keyvalue):
707 """Locate a particular node by its key property and return its id.
709 If this class has no key property, a TypeError is raised. If the
710 'keyvalue' matches one of the values for the key property among
711 the nodes in this class, the matching node's id is returned;
712 otherwise a KeyError is raised.
713 """
714 cldb = self.db.getclassdb(self.classname)
715 for nodeid in self.db.getnodeids(self.classname, cldb):
716 node = self.db.getnode(self.classname, nodeid, cldb)
717 if node.has_key(self.db.RETIRED_FLAG):
718 continue
719 if node[self.key] == keyvalue:
720 return nodeid
721 raise KeyError, keyvalue
723 # XXX: change from spec - allows multiple props to match
724 def find(self, **propspec):
725 """Get the ids of nodes in this class which link to a given node.
727 'propspec' consists of keyword args propname=nodeid
728 'propname' must be the name of a property in this class, or a
729 KeyError is raised. That property must be a Link or Multilink
730 property, or a TypeError is raised.
732 'nodeid' must be the id of an existing node in the class linked
733 to by the given property, or an IndexError is raised.
734 """
735 propspec = propspec.items()
736 for propname, nodeid in propspec:
737 # check the prop is OK
738 prop = self.properties[propname]
739 if not isinstance(prop, Link) and not isinstance(prop, Multilink):
740 raise TypeError, "'%s' not a Link/Multilink property"%propname
741 if not self.db.hasnode(prop.classname, nodeid):
742 raise ValueError, '%s has no node %s'%(prop.classname, nodeid)
744 # ok, now do the find
745 cldb = self.db.getclassdb(self.classname)
746 l = []
747 for id in self.db.getnodeids(self.classname, cldb):
748 node = self.db.getnode(self.classname, id, cldb)
749 if node.has_key(self.db.RETIRED_FLAG):
750 continue
751 for propname, nodeid in propspec:
752 property = node[propname]
753 if isinstance(prop, Link) and nodeid == property:
754 l.append(id)
755 elif isinstance(prop, Multilink) and nodeid in property:
756 l.append(id)
757 return l
759 def stringFind(self, **requirements):
760 """Locate a particular node by matching a set of its String
761 properties in a caseless search.
763 If the property is not a String property, a TypeError is raised.
765 The return is a list of the id of all nodes that match.
766 """
767 for propname in requirements.keys():
768 prop = self.properties[propname]
769 if isinstance(not prop, String):
770 raise TypeError, "'%s' not a String property"%propname
771 requirements[propname] = requirements[propname].lower()
772 l = []
773 cldb = self.db.getclassdb(self.classname)
774 for nodeid in self.db.getnodeids(self.classname, cldb):
775 node = self.db.getnode(self.classname, nodeid, cldb)
776 if node.has_key(self.db.RETIRED_FLAG):
777 continue
778 for key, value in requirements.items():
779 if node[key] and node[key].lower() != value:
780 break
781 else:
782 l.append(nodeid)
783 return l
785 def list(self):
786 """Return a list of the ids of the active nodes in this class."""
787 l = []
788 cn = self.classname
789 cldb = self.db.getclassdb(cn)
790 for nodeid in self.db.getnodeids(cn, cldb):
791 node = self.db.getnode(cn, nodeid, cldb)
792 if node.has_key(self.db.RETIRED_FLAG):
793 continue
794 l.append(nodeid)
795 l.sort()
796 return l
798 # XXX not in spec
799 def filter(self, filterspec, sort, group, num_re = re.compile('^\d+$')):
800 ''' Return a list of the ids of the active nodes in this class that
801 match the 'filter' spec, sorted by the group spec and then the
802 sort spec
803 '''
804 cn = self.classname
806 # optimise filterspec
807 l = []
808 props = self.getprops()
809 for k, v in filterspec.items():
810 propclass = props[k]
811 if isinstance(propclass, Link):
812 if type(v) is not type([]):
813 v = [v]
814 # replace key values with node ids
815 u = []
816 link_class = self.db.classes[propclass.classname]
817 for entry in v:
818 if entry == '-1': entry = None
819 elif not num_re.match(entry):
820 try:
821 entry = link_class.lookup(entry)
822 except (TypeError,KeyError):
823 raise ValueError, 'property "%s": %s not a %s'%(
824 k, entry, self.properties[k].classname)
825 u.append(entry)
827 l.append((0, k, u))
828 elif isinstance(propclass, Multilink):
829 if type(v) is not type([]):
830 v = [v]
831 # replace key values with node ids
832 u = []
833 link_class = self.db.classes[propclass.classname]
834 for entry in v:
835 if not num_re.match(entry):
836 try:
837 entry = link_class.lookup(entry)
838 except (TypeError,KeyError):
839 raise ValueError, 'new property "%s": %s not a %s'%(
840 k, entry, self.properties[k].classname)
841 u.append(entry)
842 l.append((1, k, u))
843 elif isinstance(propclass, String):
844 # simple glob searching
845 v = re.sub(r'([\|\{\}\\\.\+\[\]\(\)])', r'\\\1', v)
846 v = v.replace('?', '.')
847 v = v.replace('*', '.*?')
848 l.append((2, k, re.compile(v, re.I)))
849 else:
850 l.append((6, k, v))
851 filterspec = l
853 # now, find all the nodes that are active and pass filtering
854 l = []
855 cldb = self.db.getclassdb(cn)
856 for nodeid in self.db.getnodeids(cn, cldb):
857 node = self.db.getnode(cn, nodeid, cldb)
858 if node.has_key(self.db.RETIRED_FLAG):
859 continue
860 # apply filter
861 for t, k, v in filterspec:
862 # this node doesn't have this property, so reject it
863 if not node.has_key(k): break
865 if t == 0 and node[k] not in v:
866 # link - if this node'd property doesn't appear in the
867 # filterspec's nodeid list, skip it
868 break
869 elif t == 1:
870 # multilink - if any of the nodeids required by the
871 # filterspec aren't in this node's property, then skip
872 # it
873 for value in v:
874 if value not in node[k]:
875 break
876 else:
877 continue
878 break
879 elif t == 2 and (node[k] is None or not v.search(node[k])):
880 # RE search
881 break
882 elif t == 6 and node[k] != v:
883 # straight value comparison for the other types
884 break
885 else:
886 l.append((nodeid, node))
887 l.sort()
889 # optimise sort
890 m = []
891 for entry in sort:
892 if entry[0] != '-':
893 m.append(('+', entry))
894 else:
895 m.append((entry[0], entry[1:]))
896 sort = m
898 # optimise group
899 m = []
900 for entry in group:
901 if entry[0] != '-':
902 m.append(('+', entry))
903 else:
904 m.append((entry[0], entry[1:]))
905 group = m
906 # now, sort the result
907 def sortfun(a, b, sort=sort, group=group, properties=self.getprops(),
908 db = self.db, cl=self):
909 a_id, an = a
910 b_id, bn = b
911 # sort by group and then sort
912 for list in group, sort:
913 for dir, prop in list:
914 # sorting is class-specific
915 propclass = properties[prop]
917 # handle the properties that might be "faked"
918 # also, handle possible missing properties
919 try:
920 if not an.has_key(prop):
921 an[prop] = cl.get(a_id, prop)
922 av = an[prop]
923 except KeyError:
924 # the node doesn't have a value for this property
925 if isinstance(propclass, Multilink): av = []
926 else: av = ''
927 try:
928 if not bn.has_key(prop):
929 bn[prop] = cl.get(b_id, prop)
930 bv = bn[prop]
931 except KeyError:
932 # the node doesn't have a value for this property
933 if isinstance(propclass, Multilink): bv = []
934 else: bv = ''
936 # String and Date values are sorted in the natural way
937 if isinstance(propclass, String):
938 # clean up the strings
939 if av and av[0] in string.uppercase:
940 av = an[prop] = av.lower()
941 if bv and bv[0] in string.uppercase:
942 bv = bn[prop] = bv.lower()
943 if (isinstance(propclass, String) or
944 isinstance(propclass, Date)):
945 # it might be a string that's really an integer
946 try:
947 av = int(av)
948 bv = int(bv)
949 except:
950 pass
951 if dir == '+':
952 r = cmp(av, bv)
953 if r != 0: return r
954 elif dir == '-':
955 r = cmp(bv, av)
956 if r != 0: return r
958 # Link properties are sorted according to the value of
959 # the "order" property on the linked nodes if it is
960 # present; or otherwise on the key string of the linked
961 # nodes; or finally on the node ids.
962 elif isinstance(propclass, Link):
963 link = db.classes[propclass.classname]
964 if av is None and bv is not None: return -1
965 if av is not None and bv is None: return 1
966 if av is None and bv is None: continue
967 if link.getprops().has_key('order'):
968 if dir == '+':
969 r = cmp(link.get(av, 'order'),
970 link.get(bv, 'order'))
971 if r != 0: return r
972 elif dir == '-':
973 r = cmp(link.get(bv, 'order'),
974 link.get(av, 'order'))
975 if r != 0: return r
976 elif link.getkey():
977 key = link.getkey()
978 if dir == '+':
979 r = cmp(link.get(av, key), link.get(bv, key))
980 if r != 0: return r
981 elif dir == '-':
982 r = cmp(link.get(bv, key), link.get(av, key))
983 if r != 0: return r
984 else:
985 if dir == '+':
986 r = cmp(av, bv)
987 if r != 0: return r
988 elif dir == '-':
989 r = cmp(bv, av)
990 if r != 0: return r
992 # Multilink properties are sorted according to how many
993 # links are present.
994 elif isinstance(propclass, Multilink):
995 if dir == '+':
996 r = cmp(len(av), len(bv))
997 if r != 0: return r
998 elif dir == '-':
999 r = cmp(len(bv), len(av))
1000 if r != 0: return r
1001 # end for dir, prop in list:
1002 # end for list in sort, group:
1003 # if all else fails, compare the ids
1004 return cmp(a[0], b[0])
1006 l.sort(sortfun)
1007 return [i[0] for i in l]
1009 def count(self):
1010 """Get the number of nodes in this class.
1012 If the returned integer is 'numnodes', the ids of all the nodes
1013 in this class run from 1 to numnodes, and numnodes+1 will be the
1014 id of the next node to be created in this class.
1015 """
1016 return self.db.countnodes(self.classname)
1018 # Manipulating properties:
1020 def getprops(self, protected=1):
1021 """Return a dictionary mapping property names to property objects.
1022 If the "protected" flag is true, we include protected properties -
1023 those which may not be modified."""
1024 d = self.properties.copy()
1025 if protected:
1026 d['id'] = String()
1027 return d
1029 def addprop(self, **properties):
1030 """Add properties to this class.
1032 The keyword arguments in 'properties' must map names to property
1033 objects, or a TypeError is raised. None of the keys in 'properties'
1034 may collide with the names of existing properties, or a ValueError
1035 is raised before any properties have been added.
1036 """
1037 for key in properties.keys():
1038 if self.properties.has_key(key):
1039 raise ValueError, key
1040 self.properties.update(properties)
1042 # XXX not in spec
1043 class Node:
1044 ''' A convenience wrapper for the given node
1045 '''
1046 def __init__(self, cl, nodeid, cache=1):
1047 self.__dict__['cl'] = cl
1048 self.__dict__['nodeid'] = nodeid
1049 self.__dict__['cache'] = cache
1050 def keys(self, protected=1):
1051 return self.cl.getprops(protected=protected).keys()
1052 def values(self, protected=1):
1053 l = []
1054 for name in self.cl.getprops(protected=protected).keys():
1055 l.append(self.cl.get(self.nodeid, name, cache=self.cache))
1056 return l
1057 def items(self, protected=1):
1058 l = []
1059 for name in self.cl.getprops(protected=protected).keys():
1060 l.append((name, self.cl.get(self.nodeid, name, cache=self.cache)))
1061 return l
1062 def has_key(self, name):
1063 return self.cl.getprops().has_key(name)
1064 def __getattr__(self, name):
1065 if self.__dict__.has_key(name):
1066 return self.__dict__[name]
1067 try:
1068 return self.cl.get(self.nodeid, name, cache=self.cache)
1069 except KeyError, value:
1070 # we trap this but re-raise it as AttributeError - all other
1071 # exceptions should pass through untrapped
1072 pass
1073 # nope, no such attribute
1074 raise AttributeError, str(value)
1075 def __getitem__(self, name):
1076 return self.cl.get(self.nodeid, name, cache=self.cache)
1077 def __setattr__(self, name, value):
1078 try:
1079 return self.cl.set(self.nodeid, **{name: value})
1080 except KeyError, value:
1081 raise AttributeError, str(value)
1082 def __setitem__(self, name, value):
1083 self.cl.set(self.nodeid, **{name: value})
1084 def history(self):
1085 return self.cl.history(self.nodeid)
1086 def retire(self):
1087 return self.cl.retire(self.nodeid)
1090 def Choice(name, db, *options):
1091 '''Quick helper to create a simple class with choices
1092 '''
1093 cl = Class(db, name, name=String(), order=String())
1094 for i in range(len(options)):
1095 cl.create(name=options[i], order=i)
1096 return hyperdb.Link(name)
1098 #
1099 # $Log: not supported by cvs2svn $
1100 # Revision 1.58 2002/02/27 03:23:16 richard
1101 # Ran it through pychecker, made fixes
1102 #
1103 # Revision 1.57 2002/02/20 05:23:24 richard
1104 # Didn't accomodate new values for new properties
1105 #
1106 # Revision 1.56 2002/02/20 05:05:28 richard
1107 # . Added simple editing for classes that don't define a templated interface.
1108 # - access using the admin "class list" interface
1109 # - limited to admin-only
1110 # - requires the csv module from object-craft (url given if it's missing)
1111 #
1112 # Revision 1.55 2002/02/15 07:27:12 richard
1113 # Oops, precedences around the way w0rng.
1114 #
1115 # Revision 1.54 2002/02/15 07:08:44 richard
1116 # . Alternate email addresses are now available for users. See the MIGRATION
1117 # file for info on how to activate the feature.
1118 #
1119 # Revision 1.53 2002/01/22 07:21:13 richard
1120 # . fixed back_bsddb so it passed the journal tests
1121 #
1122 # ... it didn't seem happy using the back_anydbm _open method, which is odd.
1123 # Yet another occurrance of whichdb not being able to recognise older bsddb
1124 # databases. Yadda yadda. Made the HYPERDBDEBUG stuff more sane in the
1125 # process.
1126 #
1127 # Revision 1.52 2002/01/21 16:33:19 rochecompaan
1128 # You can now use the roundup-admin tool to pack the database
1129 #
1130 # Revision 1.51 2002/01/21 03:01:29 richard
1131 # brief docco on the do_journal argument
1132 #
1133 # Revision 1.50 2002/01/19 13:16:04 rochecompaan
1134 # Journal entries for link and multilink properties can now be switched on
1135 # or off.
1136 #
1137 # Revision 1.49 2002/01/16 07:02:57 richard
1138 # . lots of date/interval related changes:
1139 # - more relaxed date format for input
1140 #
1141 # Revision 1.48 2002/01/14 06:32:34 richard
1142 # . #502951 ] adding new properties to old database
1143 #
1144 # Revision 1.47 2002/01/14 02:20:15 richard
1145 # . changed all config accesses so they access either the instance or the
1146 # config attriubute on the db. This means that all config is obtained from
1147 # instance_config instead of the mish-mash of classes. This will make
1148 # switching to a ConfigParser setup easier too, I hope.
1149 #
1150 # At a minimum, this makes migration a _little_ easier (a lot easier in the
1151 # 0.5.0 switch, I hope!)
1152 #
1153 # Revision 1.46 2002/01/07 10:42:23 richard
1154 # oops
1155 #
1156 # Revision 1.45 2002/01/02 04:18:17 richard
1157 # hyperdb docstrings
1158 #
1159 # Revision 1.44 2002/01/02 02:31:38 richard
1160 # Sorry for the huge checkin message - I was only intending to implement #496356
1161 # but I found a number of places where things had been broken by transactions:
1162 # . modified ROUNDUPDBSENDMAILDEBUG to be SENDMAILDEBUG and hold a filename
1163 # for _all_ roundup-generated smtp messages to be sent to.
1164 # . the transaction cache had broken the roundupdb.Class set() reactors
1165 # . newly-created author users in the mailgw weren't being committed to the db
1166 #
1167 # Stuff that made it into CHANGES.txt (ie. the stuff I was actually working
1168 # on when I found that stuff :):
1169 # . #496356 ] Use threading in messages
1170 # . detectors were being registered multiple times
1171 # . added tests for mailgw
1172 # . much better attaching of erroneous messages in the mail gateway
1173 #
1174 # Revision 1.43 2001/12/20 06:13:24 rochecompaan
1175 # Bugs fixed:
1176 # . Exception handling in hyperdb for strings-that-look-like numbers got
1177 # lost somewhere
1178 # . Internet Explorer submits full path for filename - we now strip away
1179 # the path
1180 # Features added:
1181 # . Link and multilink properties are now displayed sorted in the cgi
1182 # interface
1183 #
1184 # Revision 1.42 2001/12/16 10:53:37 richard
1185 # take a copy of the node dict so that the subsequent set
1186 # operation doesn't modify the oldvalues structure
1187 #
1188 # Revision 1.41 2001/12/15 23:47:47 richard
1189 # Cleaned up some bare except statements
1190 #
1191 # Revision 1.40 2001/12/14 23:42:57 richard
1192 # yuck, a gdbm instance tests false :(
1193 # I've left the debugging code in - it should be removed one day if we're ever
1194 # _really_ anal about performace :)
1195 #
1196 # Revision 1.39 2001/12/02 05:06:16 richard
1197 # . We now use weakrefs in the Classes to keep the database reference, so
1198 # the close() method on the database is no longer needed.
1199 # I bumped the minimum python requirement up to 2.1 accordingly.
1200 # . #487480 ] roundup-server
1201 # . #487476 ] INSTALL.txt
1202 #
1203 # I also cleaned up the change message / post-edit stuff in the cgi client.
1204 # There's now a clearly marked "TODO: append the change note" where I believe
1205 # the change note should be added there. The "changes" list will obviously
1206 # have to be modified to be a dict of the changes, or somesuch.
1207 #
1208 # More testing needed.
1209 #
1210 # Revision 1.38 2001/12/01 07:17:50 richard
1211 # . We now have basic transaction support! Information is only written to
1212 # the database when the commit() method is called. Only the anydbm
1213 # backend is modified in this way - neither of the bsddb backends have been.
1214 # The mail, admin and cgi interfaces all use commit (except the admin tool
1215 # doesn't have a commit command, so interactive users can't commit...)
1216 # . Fixed login/registration forwarding the user to the right page (or not,
1217 # on a failure)
1218 #
1219 # Revision 1.37 2001/11/28 21:55:35 richard
1220 # . login_action and newuser_action return values were being ignored
1221 # . Woohoo! Found that bloody re-login bug that was killing the mail
1222 # gateway.
1223 # (also a minor cleanup in hyperdb)
1224 #
1225 # Revision 1.36 2001/11/27 03:16:09 richard
1226 # Another place that wasn't handling missing properties.
1227 #
1228 # Revision 1.35 2001/11/22 15:46:42 jhermann
1229 # Added module docstrings to all modules.
1230 #
1231 # Revision 1.34 2001/11/21 04:04:43 richard
1232 # *sigh* more missing value handling
1233 #
1234 # Revision 1.33 2001/11/21 03:40:54 richard
1235 # more new property handling
1236 #
1237 # Revision 1.32 2001/11/21 03:11:28 richard
1238 # Better handling of new properties.
1239 #
1240 # Revision 1.31 2001/11/12 22:01:06 richard
1241 # Fixed issues with nosy reaction and author copies.
1242 #
1243 # Revision 1.30 2001/11/09 10:11:08 richard
1244 # . roundup-admin now handles all hyperdb exceptions
1245 #
1246 # Revision 1.29 2001/10/27 00:17:41 richard
1247 # Made Class.stringFind() do caseless matching.
1248 #
1249 # Revision 1.28 2001/10/21 04:44:50 richard
1250 # bug #473124: UI inconsistency with Link fields.
1251 # This also prompted me to fix a fairly long-standing usability issue -
1252 # that of being able to turn off certain filters.
1253 #
1254 # Revision 1.27 2001/10/20 23:44:27 richard
1255 # Hyperdatabase sorts strings-that-look-like-numbers as numbers now.
1256 #
1257 # Revision 1.26 2001/10/16 03:48:01 richard
1258 # admin tool now complains if a "find" is attempted with a non-link property.
1259 #
1260 # Revision 1.25 2001/10/11 00:17:51 richard
1261 # Reverted a change in hyperdb so the default value for missing property
1262 # values in a create() is None and not '' (the empty string.) This obviously
1263 # breaks CSV import/export - the string 'None' will be created in an
1264 # export/import operation.
1265 #
1266 # Revision 1.24 2001/10/10 03:54:57 richard
1267 # Added database importing and exporting through CSV files.
1268 # Uses the csv module from object-craft for exporting if it's available.
1269 # Requires the csv module for importing.
1270 #
1271 # Revision 1.23 2001/10/09 23:58:10 richard
1272 # Moved the data stringification up into the hyperdb.Class class' get, set
1273 # and create methods. This means that the data is also stringified for the
1274 # journal call, and removes duplication of code from the backends. The
1275 # backend code now only sees strings.
1276 #
1277 # Revision 1.22 2001/10/09 07:25:59 richard
1278 # Added the Password property type. See "pydoc roundup.password" for
1279 # implementation details. Have updated some of the documentation too.
1280 #
1281 # Revision 1.21 2001/10/05 02:23:24 richard
1282 # . roundup-admin create now prompts for property info if none is supplied
1283 # on the command-line.
1284 # . hyperdb Class getprops() method may now return only the mutable
1285 # properties.
1286 # . Login now uses cookies, which makes it a whole lot more flexible. We can
1287 # now support anonymous user access (read-only, unless there's an
1288 # "anonymous" user, in which case write access is permitted). Login
1289 # handling has been moved into cgi_client.Client.main()
1290 # . The "extended" schema is now the default in roundup init.
1291 # . The schemas have had their page headings modified to cope with the new
1292 # login handling. Existing installations should copy the interfaces.py
1293 # file from the roundup lib directory to their instance home.
1294 # . Incorrectly had a Bizar Software copyright on the cgitb.py module from
1295 # Ping - has been removed.
1296 # . Fixed a whole bunch of places in the CGI interface where we should have
1297 # been returning Not Found instead of throwing an exception.
1298 # . Fixed a deviation from the spec: trying to modify the 'id' property of
1299 # an item now throws an exception.
1300 #
1301 # Revision 1.20 2001/10/04 02:12:42 richard
1302 # Added nicer command-line item adding: passing no arguments will enter an
1303 # interactive more which asks for each property in turn. While I was at it, I
1304 # fixed an implementation problem WRT the spec - I wasn't raising a
1305 # ValueError if the key property was missing from a create(). Also added a
1306 # protected=boolean argument to getprops() so we can list only the mutable
1307 # properties (defaults to yes, which lists the immutables).
1308 #
1309 # Revision 1.19 2001/08/29 04:47:18 richard
1310 # Fixed CGI client change messages so they actually include the properties
1311 # changed (again).
1312 #
1313 # Revision 1.18 2001/08/16 07:34:59 richard
1314 # better CGI text searching - but hidden filter fields are disappearing...
1315 #
1316 # Revision 1.17 2001/08/16 06:59:58 richard
1317 # all searches use re now - and they're all case insensitive
1318 #
1319 # Revision 1.16 2001/08/15 23:43:18 richard
1320 # Fixed some isFooTypes that I missed.
1321 # Refactored some code in the CGI code.
1322 #
1323 # Revision 1.15 2001/08/12 06:32:36 richard
1324 # using isinstance(blah, Foo) now instead of isFooType
1325 #
1326 # Revision 1.14 2001/08/07 00:24:42 richard
1327 # stupid typo
1328 #
1329 # Revision 1.13 2001/08/07 00:15:51 richard
1330 # Added the copyright/license notice to (nearly) all files at request of
1331 # Bizar Software.
1332 #
1333 # Revision 1.12 2001/08/02 06:38:17 richard
1334 # Roundupdb now appends "mailing list" information to its messages which
1335 # include the e-mail address and web interface address. Templates may
1336 # override this in their db classes to include specific information (support
1337 # instructions, etc).
1338 #
1339 # Revision 1.11 2001/08/01 04:24:21 richard
1340 # mailgw was assuming certain properties existed on the issues being created.
1341 #
1342 # Revision 1.10 2001/07/30 02:38:31 richard
1343 # get() now has a default arg - for migration only.
1344 #
1345 # Revision 1.9 2001/07/29 09:28:23 richard
1346 # Fixed sorting by clicking on column headings.
1347 #
1348 # Revision 1.8 2001/07/29 08:27:40 richard
1349 # Fixed handling of passed-in values in form elements (ie. during a
1350 # drill-down)
1351 #
1352 # Revision 1.7 2001/07/29 07:01:39 richard
1353 # Added vim command to all source so that we don't get no steenkin' tabs :)
1354 #
1355 # Revision 1.6 2001/07/29 05:36:14 richard
1356 # Cleanup of the link label generation.
1357 #
1358 # Revision 1.5 2001/07/29 04:05:37 richard
1359 # Added the fabricated property "id".
1360 #
1361 # Revision 1.4 2001/07/27 06:25:35 richard
1362 # Fixed some of the exceptions so they're the right type.
1363 # Removed the str()-ification of node ids so we don't mask oopsy errors any
1364 # more.
1365 #
1366 # Revision 1.3 2001/07/27 05:17:14 richard
1367 # just some comments
1368 #
1369 # Revision 1.2 2001/07/22 12:09:32 richard
1370 # Final commit of Grande Splite
1371 #
1372 # Revision 1.1 2001/07/22 11:58:35 richard
1373 # More Grande Splite
1374 #
1375 #
1376 # vim: set filetype=python ts=4 sw=4 et si