1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 # $Id: hyperdb.py,v 1.91 2003-11-11 00:35:13 richard Exp $
20 """
21 Hyperdatabase implementation, especially field types.
22 """
24 # standard python modules
25 import sys, os, time, re
27 # roundup modules
28 import date, password
30 # configure up the DEBUG and TRACE captures
31 class Sink:
32 def write(self, content):
33 pass
34 DEBUG = os.environ.get('HYPERDBDEBUG', '')
35 if DEBUG and __debug__:
36 if DEBUG == 'stdout':
37 DEBUG = sys.stdout
38 else:
39 DEBUG = open(DEBUG, 'a')
40 else:
41 DEBUG = Sink()
42 TRACE = os.environ.get('HYPERDBTRACE', '')
43 if TRACE and __debug__:
44 if TRACE == 'stdout':
45 TRACE = sys.stdout
46 else:
47 TRACE = open(TRACE, 'w')
48 else:
49 TRACE = Sink()
50 def traceMark():
51 print >>TRACE, '**MARK', time.ctime()
52 del Sink
54 #
55 # Types
56 #
57 class String:
58 """An object designating a String property."""
59 def __init__(self, indexme='no'):
60 self.indexme = indexme == 'yes'
61 def __repr__(self):
62 ' more useful for dumps '
63 return '<%s>'%self.__class__
65 class Password:
66 """An object designating a Password property."""
67 def __repr__(self):
68 ' more useful for dumps '
69 return '<%s>'%self.__class__
71 class Date:
72 """An object designating a Date property."""
73 def __repr__(self):
74 ' more useful for dumps '
75 return '<%s>'%self.__class__
77 class Interval:
78 """An object designating an Interval property."""
79 def __repr__(self):
80 ' more useful for dumps '
81 return '<%s>'%self.__class__
83 class Link:
84 """An object designating a Link property that links to a
85 node in a specified class."""
86 def __init__(self, classname, do_journal='yes'):
87 ''' Default is to not journal link and unlink events
88 '''
89 self.classname = classname
90 self.do_journal = do_journal == 'yes'
91 def __repr__(self):
92 ' more useful for dumps '
93 return '<%s to "%s">'%(self.__class__, self.classname)
95 class Multilink:
96 """An object designating a Multilink property that links
97 to nodes in a specified class.
99 "classname" indicates the class to link to
101 "do_journal" indicates whether the linked-to nodes should have
102 'link' and 'unlink' events placed in their journal
103 """
104 def __init__(self, classname, do_journal='yes'):
105 ''' Default is to not journal link and unlink events
106 '''
107 self.classname = classname
108 self.do_journal = do_journal == 'yes'
109 def __repr__(self):
110 ' more useful for dumps '
111 return '<%s to "%s">'%(self.__class__, self.classname)
113 class Boolean:
114 """An object designating a boolean property"""
115 def __repr__(self):
116 'more useful for dumps'
117 return '<%s>' % self.__class__
119 class Number:
120 """An object designating a numeric property"""
121 def __repr__(self):
122 'more useful for dumps'
123 return '<%s>' % self.__class__
124 #
125 # Support for splitting designators
126 #
127 class DesignatorError(ValueError):
128 pass
129 def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')):
130 ''' Take a foo123 and return ('foo', 123)
131 '''
132 m = dre.match(designator)
133 if m is None:
134 raise DesignatorError, '"%s" not a node designator'%designator
135 return m.group(1), m.group(2)
137 #
138 # the base Database class
139 #
140 class DatabaseError(ValueError):
141 '''Error to be raised when there is some problem in the database code
142 '''
143 pass
144 class Database:
145 '''A database for storing records containing flexible data types.
147 This class defines a hyperdatabase storage layer, which the Classes use to
148 store their data.
151 Transactions
152 ------------
153 The Database should support transactions through the commit() and
154 rollback() methods. All other Database methods should be transaction-aware,
155 using data from the current transaction before looking up the database.
157 An implementation must provide an override for the get() method so that the
158 in-database value is returned in preference to the in-transaction value.
159 This is necessary to determine if any values have changed during a
160 transaction.
163 Implementation
164 --------------
166 All methods except __repr__ and getnode must be implemented by a
167 concrete backend Class.
169 '''
171 # flag to set on retired entries
172 RETIRED_FLAG = '__hyperdb_retired'
174 def __init__(self, config, journaltag=None):
175 """Open a hyperdatabase given a specifier to some storage.
177 The 'storagelocator' is obtained from config.DATABASE.
178 The meaning of 'storagelocator' depends on the particular
179 implementation of the hyperdatabase. It could be a file name,
180 a directory path, a socket descriptor for a connection to a
181 database over the network, etc.
183 The 'journaltag' is a token that will be attached to the journal
184 entries for any edits done on the database. If 'journaltag' is
185 None, the database is opened in read-only mode: the Class.create(),
186 Class.set(), and Class.retire() methods are disabled.
187 """
188 raise NotImplementedError
190 def post_init(self):
191 """Called once the schema initialisation has finished.
192 If 'refresh' is true, we want to rebuild the backend
193 structures.
194 """
195 raise NotImplementedError
197 def refresh_database(self):
198 """Called to indicate that the backend should rebuild all tables
199 and structures. Not called in normal usage."""
200 raise NotImplementedError
202 def __getattr__(self, classname):
203 """A convenient way of calling self.getclass(classname)."""
204 raise NotImplementedError
206 def addclass(self, cl):
207 '''Add a Class to the hyperdatabase.
208 '''
209 raise NotImplementedError
211 def getclasses(self):
212 """Return a list of the names of all existing classes."""
213 raise NotImplementedError
215 def getclass(self, classname):
216 """Get the Class object representing a particular class.
218 If 'classname' is not a valid class name, a KeyError is raised.
219 """
220 raise NotImplementedError
222 def clear(self):
223 '''Delete all database contents.
224 '''
225 raise NotImplementedError
227 def getclassdb(self, classname, mode='r'):
228 '''Obtain a connection to the class db that will be used for
229 multiple actions.
230 '''
231 raise NotImplementedError
233 def addnode(self, classname, nodeid, node):
234 '''Add the specified node to its class's db.
235 '''
236 raise NotImplementedError
238 def serialise(self, classname, node):
239 '''Copy the node contents, converting non-marshallable data into
240 marshallable data.
241 '''
242 return node
244 def setnode(self, classname, nodeid, node):
245 '''Change the specified node.
246 '''
247 raise NotImplementedError
249 def unserialise(self, classname, node):
250 '''Decode the marshalled node data
251 '''
252 return node
254 def getnode(self, classname, nodeid, db=None, cache=1):
255 '''Get a node from the database.
257 'cache' exists for backwards compatibility, and is not used.
258 '''
259 raise NotImplementedError
261 def hasnode(self, classname, nodeid, db=None):
262 '''Determine if the database has a given node.
263 '''
264 raise NotImplementedError
266 def countnodes(self, classname, db=None):
267 '''Count the number of nodes that exist for a particular Class.
268 '''
269 raise NotImplementedError
271 def storefile(self, classname, nodeid, property, content):
272 '''Store the content of the file in the database.
274 The property may be None, in which case the filename does not
275 indicate which property is being saved.
276 '''
277 raise NotImplementedError
279 def getfile(self, classname, nodeid, property):
280 '''Store the content of the file in the database.
281 '''
282 raise NotImplementedError
284 def addjournal(self, classname, nodeid, action, params):
285 ''' Journal the Action
286 'action' may be:
288 'create' or 'set' -- 'params' is a dictionary of property values
289 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
290 'retire' -- 'params' is None
291 '''
292 raise NotImplementedError
294 def getjournal(self, classname, nodeid):
295 ''' get the journal for id
296 '''
297 raise NotImplementedError
299 def pack(self, pack_before):
300 ''' pack the database
301 '''
302 raise NotImplementedError
304 def commit(self):
305 ''' Commit the current transactions.
307 Save all data changed since the database was opened or since the
308 last commit() or rollback().
309 '''
310 raise NotImplementedError
312 def rollback(self):
313 ''' Reverse all actions from the current transaction.
315 Undo all the changes made since the database was opened or the last
316 commit() or rollback() was performed.
317 '''
318 raise NotImplementedError
320 #
321 # The base Class class
322 #
323 class Class:
324 """ The handle to a particular class of nodes in a hyperdatabase.
326 All methods except __repr__ and getnode must be implemented by a
327 concrete backend Class.
328 """
330 def __init__(self, db, classname, **properties):
331 """Create a new class with a given name and property specification.
333 'classname' must not collide with the name of an existing class,
334 or a ValueError is raised. The keyword arguments in 'properties'
335 must map names to property objects, or a TypeError is raised.
336 """
337 raise NotImplementedError
339 def __repr__(self):
340 '''Slightly more useful representation
341 '''
342 return '<hyperdb.Class "%s">'%self.classname
344 # Editing nodes:
346 def create(self, **propvalues):
347 """Create a new node of this class and return its id.
349 The keyword arguments in 'propvalues' map property names to values.
351 The values of arguments must be acceptable for the types of their
352 corresponding properties or a TypeError is raised.
354 If this class has a key property, it must be present and its value
355 must not collide with other key strings or a ValueError is raised.
357 Any other properties on this class that are missing from the
358 'propvalues' dictionary are set to None.
360 If an id in a link or multilink property does not refer to a valid
361 node, an IndexError is raised.
362 """
363 raise NotImplementedError
365 _marker = []
366 def get(self, nodeid, propname, default=_marker, cache=1):
367 """Get the value of a property on an existing node of this class.
369 'nodeid' must be the id of an existing node of this class or an
370 IndexError is raised. 'propname' must be the name of a property
371 of this class or a KeyError is raised.
373 'cache' exists for backwards compatibility, and is not used.
374 """
375 raise NotImplementedError
377 def getnode(self, nodeid, cache=1):
378 ''' Return a convenience wrapper for the node.
380 'nodeid' must be the id of an existing node of this class or an
381 IndexError is raised.
383 'cache' exists for backwards compatibility, and is not used.
384 '''
385 return Node(self, nodeid)
387 def getnodeids(self, db=None):
388 '''Retrieve all the ids of the nodes for a particular Class.
389 '''
390 raise NotImplementedError
392 def set(self, nodeid, **propvalues):
393 """Modify a property on an existing node of this class.
395 'nodeid' must be the id of an existing node of this class or an
396 IndexError is raised.
398 Each key in 'propvalues' must be the name of a property of this
399 class or a KeyError is raised.
401 All values in 'propvalues' must be acceptable types for their
402 corresponding properties or a TypeError is raised.
404 If the value of the key property is set, it must not collide with
405 other key strings or a ValueError is raised.
407 If the value of a Link or Multilink property contains an invalid
408 node id, a ValueError is raised.
409 """
410 raise NotImplementedError
412 def retire(self, nodeid):
413 """Retire a node.
415 The properties on the node remain available from the get() method,
416 and the node's id is never reused.
418 Retired nodes are not returned by the find(), list(), or lookup()
419 methods, and other nodes may reuse the values of their key properties.
420 """
421 raise NotImplementedError
423 def restore(self, nodeid):
424 '''Restpre a retired node.
426 Make node available for all operations like it was before retirement.
427 '''
428 raise NotImplementedError
430 def is_retired(self, nodeid):
431 '''Return true if the node is rerired
432 '''
433 raise NotImplementedError
435 def destroy(self, nodeid):
436 """Destroy a node.
438 WARNING: this method should never be used except in extremely rare
439 situations where there could never be links to the node being
440 deleted
441 WARNING: use retire() instead
442 WARNING: the properties of this node will not be available ever again
443 WARNING: really, use retire() instead
445 Well, I think that's enough warnings. This method exists mostly to
446 support the session storage of the cgi interface.
448 The node is completely removed from the hyperdb, including all journal
449 entries. It will no longer be available, and will generally break code
450 if there are any references to the node.
451 """
453 def history(self, nodeid):
454 """Retrieve the journal of edits on a particular node.
456 'nodeid' must be the id of an existing node of this class or an
457 IndexError is raised.
459 The returned list contains tuples of the form
461 (date, tag, action, params)
463 'date' is a Timestamp object specifying the time of the change and
464 'tag' is the journaltag specified when the database was opened.
465 """
466 raise NotImplementedError
468 # Locating nodes:
469 def hasnode(self, nodeid):
470 '''Determine if the given nodeid actually exists
471 '''
472 raise NotImplementedError
474 def setkey(self, propname):
475 """Select a String property of this class to be the key property.
477 'propname' must be the name of a String property of this class or
478 None, or a TypeError is raised. The values of the key property on
479 all existing nodes must be unique or a ValueError is raised.
480 """
481 raise NotImplementedError
483 def getkey(self):
484 """Return the name of the key property for this class or None."""
485 raise NotImplementedError
487 def labelprop(self, default_to_id=0):
488 ''' Return the property name for a label for the given node.
490 This method attempts to generate a consistent label for the node.
491 It tries the following in order:
492 1. key property
493 2. "name" property
494 3. "title" property
495 4. first property from the sorted property name list
496 '''
497 raise NotImplementedError
499 def lookup(self, keyvalue):
500 """Locate a particular node by its key property and return its id.
502 If this class has no key property, a TypeError is raised. If the
503 'keyvalue' matches one of the values for the key property among
504 the nodes in this class, the matching node's id is returned;
505 otherwise a KeyError is raised.
506 """
507 raise NotImplementedError
509 def find(self, **propspec):
510 """Get the ids of nodes in this class which link to the given nodes.
512 'propspec' consists of keyword args propname={nodeid:1,}
513 'propname' must be the name of a property in this class, or a
514 KeyError is raised. That property must be a Link or Multilink
515 property, or a TypeError is raised.
517 Any node in this class whose 'propname' property links to any of the
518 nodeids will be returned. Used by the full text indexing, which knows
519 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
520 issues:
522 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
523 """
524 raise NotImplementedError
526 def filter(self, search_matches, filterspec, sort=(None,None),
527 group=(None,None)):
528 ''' Return a list of the ids of the active nodes in this class that
529 match the 'filter' spec, sorted by the group spec and then the
530 sort spec.
532 "filterspec" is {propname: value(s)}
533 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
534 and prop is a prop name or None
535 "search_matches" is {nodeid: marker}
537 The filter must match all properties specificed - but if the
538 property value to match is a list, any one of the values in the
539 list may match for that property to match.
540 '''
541 raise NotImplementedError
543 def count(self):
544 """Get the number of nodes in this class.
546 If the returned integer is 'numnodes', the ids of all the nodes
547 in this class run from 1 to numnodes, and numnodes+1 will be the
548 id of the next node to be created in this class.
549 """
550 raise NotImplementedError
552 # Manipulating properties:
553 def getprops(self, protected=1):
554 """Return a dictionary mapping property names to property objects.
555 If the "protected" flag is true, we include protected properties -
556 those which may not be modified.
557 """
558 raise NotImplementedError
560 def addprop(self, **properties):
561 """Add properties to this class.
563 The keyword arguments in 'properties' must map names to property
564 objects, or a TypeError is raised. None of the keys in 'properties'
565 may collide with the names of existing properties, or a ValueError
566 is raised before any properties have been added.
567 """
568 raise NotImplementedError
570 def index(self, nodeid):
571 '''Add (or refresh) the node to search indexes
572 '''
573 raise NotImplementedError
575 class HyperdbValueError(ValueError):
576 ''' Error converting a raw value into a Hyperdb value '''
577 pass
579 def convertLinkValue(db, propname, prop, value, idre=re.compile('\d+')):
580 ''' Convert the link value (may be id or key value) to an id value. '''
581 linkcl = db.classes[prop.classname]
582 if not idre.match(value):
583 if linkcl.getkey():
584 try:
585 value = linkcl.lookup(value)
586 except KeyError, message:
587 raise HyperdbValueError, 'property %s: %r is not a %s.'%(
588 propname, value, prop.classname)
589 else:
590 raise HyperdbValueError, 'you may only enter ID values '\
591 'for property %s'%propname
592 return value
594 def fixNewlines(text):
595 ''' Homogenise line endings.
597 Different web clients send different line ending values, but
598 other systems (eg. email) don't necessarily handle those line
599 endings. Our solution is to convert all line endings to LF.
600 '''
601 text = text.replace('\r\n', '\n')
602 return text.replace('\r', '\n')
604 def rawToHyperdb(db, klass, itemid, propname, value,
605 pwre=re.compile(r'{(\w+)}(.+)')):
606 ''' Convert the raw (user-input) value to a hyperdb-storable value. The
607 value is for the "propname" property on itemid (may be None for a
608 new item) of "klass" in "db".
610 The value is usually a string, but in the case of multilink inputs
611 it may be either a list of strings or a string with comma-separated
612 values.
613 '''
614 properties = klass.getprops()
616 # ensure it's a valid property name
617 propname = propname.strip()
618 try:
619 proptype = properties[propname]
620 except KeyError:
621 raise HyperdbValueError, '%r is not a property of %s'%(propname,
622 klass.classname)
624 # if we got a string, strip it now
625 if isinstance(value, type('')):
626 value = value.strip()
628 # convert the input value to a real property value
629 if isinstance(proptype, String):
630 # fix the CRLF/CR -> LF stuff
631 value = fixNewlines(value)
632 if isinstance(proptype, Password):
633 m = pwre.match(value)
634 if m:
635 # password is being given to us encrypted
636 p = password.Password()
637 p.scheme = m.group(1)
638 if p.scheme not in 'SHA crypt plaintext'.split():
639 raise HyperdbValueError, 'property %s: unknown encryption '\
640 'scheme %r'%(propname, p.scheme)
641 p.password = m.group(2)
642 value = p
643 else:
644 try:
645 value = password.Password(value)
646 except password.PasswordValueError, message:
647 raise HyperdbValueError, 'property %s: %s'%(propname, message)
648 elif isinstance(proptype, Date):
649 try:
650 tz = db.getUserTimezone()
651 value = date.Date(value).local(tz)
652 except ValueError, message:
653 raise HyperdbValueError, 'property %s: %r is an invalid '\
654 'date (%s)'%(propname, value, message)
655 elif isinstance(proptype, Interval):
656 try:
657 value = date.Interval(value)
658 except ValueError, message:
659 raise HyperdbValueError, 'property %s: %r is an invalid '\
660 'date interval (%s)'%(propname, value, message)
661 elif isinstance(proptype, Link):
662 if value == '-1' or not value:
663 value = None
664 else:
665 value = convertLinkValue(db, propname, proptype, value)
667 elif isinstance(proptype, Multilink):
668 # get the current item value if it's not a new item
669 if itemid and not itemid.startswith('-'):
670 curvalue = klass.get(itemid, propname)
671 else:
672 curvalue = []
674 # if the value is a comma-separated string then split it now
675 if isinstance(value, type('')):
676 value = value.split(',')
678 # handle each add/remove in turn
679 # keep an extra list for all items that are
680 # definitely in the new list (in case of e.g.
681 # <propname>=A,+B, which should replace the old
682 # list with A,B)
683 set = 1
684 newvalue = []
685 for item in value:
686 item = item.strip()
688 # skip blanks
689 if not item: continue
691 # handle +/-
692 remove = 0
693 if item.startswith('-'):
694 remove = 1
695 item = item[1:]
696 set = 0
697 elif item.startswith('+'):
698 item = item[1:]
699 set = 0
701 # look up the value
702 itemid = convertLinkValue(db, propname, proptype, item)
704 # perform the add/remove
705 if remove:
706 try:
707 curvalue.remove(itemid)
708 except ValueError:
709 raise HyperdbValueError, 'property %s: %r is not ' \
710 'currently an element'%(propname, item)
711 else:
712 newvalue.append(itemid)
713 if itemid not in curvalue:
714 curvalue.append(itemid)
716 # that's it, set the new Multilink property value,
717 # or overwrite it completely
718 if set:
719 value = newvalue
720 else:
721 value = curvalue
723 # TODO: one day, we'll switch to numeric ids and this will be
724 # unnecessary :(
725 value = [int(x) for x in value]
726 value.sort()
727 value = [str(x) for x in value]
728 elif isinstance(proptype, Boolean):
729 value = value.strip()
730 value = value.lower() in ('yes', 'true', 'on', '1')
731 elif isinstance(proptype, Number):
732 value = value.strip()
733 try:
734 value = float(value)
735 except ValueError:
736 raise HyperdbValueError, 'property %s: %r is not a number'%(
737 propname, value)
738 return value
740 class FileClass:
741 ''' A class that requires the "content" property and stores it on
742 disk.
743 '''
744 pass
746 class Node:
747 ''' A convenience wrapper for the given node
748 '''
749 def __init__(self, cl, nodeid, cache=1):
750 self.__dict__['cl'] = cl
751 self.__dict__['nodeid'] = nodeid
752 def keys(self, protected=1):
753 return self.cl.getprops(protected=protected).keys()
754 def values(self, protected=1):
755 l = []
756 for name in self.cl.getprops(protected=protected).keys():
757 l.append(self.cl.get(self.nodeid, name))
758 return l
759 def items(self, protected=1):
760 l = []
761 for name in self.cl.getprops(protected=protected).keys():
762 l.append((name, self.cl.get(self.nodeid, name)))
763 return l
764 def has_key(self, name):
765 return self.cl.getprops().has_key(name)
766 def get(self, name, default=None):
767 if self.has_key(name):
768 return self[name]
769 else:
770 return default
771 def __getattr__(self, name):
772 if self.__dict__.has_key(name):
773 return self.__dict__[name]
774 try:
775 return self.cl.get(self.nodeid, name)
776 except KeyError, value:
777 # we trap this but re-raise it as AttributeError - all other
778 # exceptions should pass through untrapped
779 pass
780 # nope, no such attribute
781 raise AttributeError, str(value)
782 def __getitem__(self, name):
783 return self.cl.get(self.nodeid, name)
784 def __setattr__(self, name, value):
785 try:
786 return self.cl.set(self.nodeid, **{name: value})
787 except KeyError, value:
788 raise AttributeError, str(value)
789 def __setitem__(self, name, value):
790 self.cl.set(self.nodeid, **{name: value})
791 def history(self):
792 return self.cl.history(self.nodeid)
793 def retire(self):
794 return self.cl.retire(self.nodeid)
797 def Choice(name, db, *options):
798 '''Quick helper to create a simple class with choices
799 '''
800 cl = Class(db, name, name=String(), order=String())
801 for i in range(len(options)):
802 cl.create(name=options[i], order=i)
803 return hyperdb.Link(name)
805 # vim: set filetype=python ts=4 sw=4 et si