4bac87fc63039f2f258e2a26737990063e101f3b
1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 # $Id: hyperdb.py,v 1.96 2004-02-11 23:55:08 richard Exp $
20 """Hyperdatabase implementation, especially field types.
21 """
22 __docformat__ = 'restructuredtext'
24 # standard python modules
25 import sys, os, time, re
27 # roundup modules
28 import date, password
30 # configure up the DEBUG and TRACE captures
31 class Sink:
32 def write(self, content):
33 pass
34 DEBUG = os.environ.get('HYPERDBDEBUG', '')
35 if DEBUG and __debug__:
36 if DEBUG == 'stdout':
37 DEBUG = sys.stdout
38 else:
39 DEBUG = open(DEBUG, 'a')
40 else:
41 DEBUG = Sink()
42 TRACE = os.environ.get('HYPERDBTRACE', '')
43 if TRACE and __debug__:
44 if TRACE == 'stdout':
45 TRACE = sys.stdout
46 else:
47 TRACE = open(TRACE, 'w')
48 else:
49 TRACE = Sink()
50 def traceMark():
51 print >>TRACE, '**MARK', time.ctime()
52 del Sink
54 #
55 # Types
56 #
57 class String:
58 """An object designating a String property."""
59 def __init__(self, indexme='no'):
60 self.indexme = indexme == 'yes'
61 def __repr__(self):
62 ' more useful for dumps '
63 return '<%s>'%self.__class__
65 class Password:
66 """An object designating a Password property."""
67 def __repr__(self):
68 ' more useful for dumps '
69 return '<%s>'%self.__class__
71 class Date:
72 """An object designating a Date property."""
73 def __repr__(self):
74 ' more useful for dumps '
75 return '<%s>'%self.__class__
77 class Interval:
78 """An object designating an Interval property."""
79 def __repr__(self):
80 ' more useful for dumps '
81 return '<%s>'%self.__class__
83 class Link:
84 """An object designating a Link property that links to a
85 node in a specified class."""
86 def __init__(self, classname, do_journal='yes'):
87 ''' Default is to not journal link and unlink events
88 '''
89 self.classname = classname
90 self.do_journal = do_journal == 'yes'
91 def __repr__(self):
92 ' more useful for dumps '
93 return '<%s to "%s">'%(self.__class__, self.classname)
95 class Multilink:
96 """An object designating a Multilink property that links
97 to nodes in a specified class.
99 "classname" indicates the class to link to
101 "do_journal" indicates whether the linked-to nodes should have
102 'link' and 'unlink' events placed in their journal
103 """
104 def __init__(self, classname, do_journal='yes'):
105 ''' Default is to not journal link and unlink events
106 '''
107 self.classname = classname
108 self.do_journal = do_journal == 'yes'
109 def __repr__(self):
110 ' more useful for dumps '
111 return '<%s to "%s">'%(self.__class__, self.classname)
113 class Boolean:
114 """An object designating a boolean property"""
115 def __repr__(self):
116 'more useful for dumps'
117 return '<%s>' % self.__class__
119 class Number:
120 """An object designating a numeric property"""
121 def __repr__(self):
122 'more useful for dumps'
123 return '<%s>' % self.__class__
124 #
125 # Support for splitting designators
126 #
127 class DesignatorError(ValueError):
128 pass
129 def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')):
130 ''' Take a foo123 and return ('foo', 123)
131 '''
132 m = dre.match(designator)
133 if m is None:
134 raise DesignatorError, '"%s" not a node designator'%designator
135 return m.group(1), m.group(2)
137 #
138 # the base Database class
139 #
140 class DatabaseError(ValueError):
141 '''Error to be raised when there is some problem in the database code
142 '''
143 pass
144 class Database:
145 '''A database for storing records containing flexible data types.
147 This class defines a hyperdatabase storage layer, which the Classes use to
148 store their data.
151 Transactions
152 ------------
153 The Database should support transactions through the commit() and
154 rollback() methods. All other Database methods should be transaction-aware,
155 using data from the current transaction before looking up the database.
157 An implementation must provide an override for the get() method so that the
158 in-database value is returned in preference to the in-transaction value.
159 This is necessary to determine if any values have changed during a
160 transaction.
163 Implementation
164 --------------
166 All methods except __repr__ must be implemented by a concrete backend Database.
168 '''
170 # flag to set on retired entries
171 RETIRED_FLAG = '__hyperdb_retired'
173 def __init__(self, config, journaltag=None):
174 """Open a hyperdatabase given a specifier to some storage.
176 The 'storagelocator' is obtained from config.DATABASE.
177 The meaning of 'storagelocator' depends on the particular
178 implementation of the hyperdatabase. It could be a file name,
179 a directory path, a socket descriptor for a connection to a
180 database over the network, etc.
182 The 'journaltag' is a token that will be attached to the journal
183 entries for any edits done on the database. If 'journaltag' is
184 None, the database is opened in read-only mode: the Class.create(),
185 Class.set(), and Class.retire() methods are disabled.
186 """
187 raise NotImplementedError
189 def post_init(self):
190 """Called once the schema initialisation has finished.
191 If 'refresh' is true, we want to rebuild the backend
192 structures.
193 """
194 raise NotImplementedError
196 def refresh_database(self):
197 """Called to indicate that the backend should rebuild all tables
198 and structures. Not called in normal usage."""
199 raise NotImplementedError
201 def __getattr__(self, classname):
202 """A convenient way of calling self.getclass(classname)."""
203 raise NotImplementedError
205 def addclass(self, cl):
206 '''Add a Class to the hyperdatabase.
207 '''
208 raise NotImplementedError
210 def getclasses(self):
211 """Return a list of the names of all existing classes."""
212 raise NotImplementedError
214 def getclass(self, classname):
215 """Get the Class object representing a particular class.
217 If 'classname' is not a valid class name, a KeyError is raised.
218 """
219 raise NotImplementedError
221 def clear(self):
222 '''Delete all database contents.
223 '''
224 raise NotImplementedError
226 def getclassdb(self, classname, mode='r'):
227 '''Obtain a connection to the class db that will be used for
228 multiple actions.
229 '''
230 raise NotImplementedError
232 def addnode(self, classname, nodeid, node):
233 """Add the specified node to its class's db.
234 """
235 raise NotImplementedError
237 def serialise(self, classname, node):
238 '''Copy the node contents, converting non-marshallable data into
239 marshallable data.
240 '''
241 return node
243 def setnode(self, classname, nodeid, node):
244 '''Change the specified node.
245 '''
246 raise NotImplementedError
248 def unserialise(self, classname, node):
249 '''Decode the marshalled node data
250 '''
251 return node
253 def getnode(self, classname, nodeid):
254 '''Get a node from the database.
256 'cache' exists for backwards compatibility, and is not used.
257 '''
258 raise NotImplementedError
260 def hasnode(self, classname, nodeid):
261 '''Determine if the database has a given node.
262 '''
263 raise NotImplementedError
265 def countnodes(self, classname):
266 '''Count the number of nodes that exist for a particular Class.
267 '''
268 raise NotImplementedError
270 def storefile(self, classname, nodeid, property, content):
271 '''Store the content of the file in the database.
273 The property may be None, in which case the filename does not
274 indicate which property is being saved.
275 '''
276 raise NotImplementedError
278 def getfile(self, classname, nodeid, property):
279 '''Store the content of the file in the database.
280 '''
281 raise NotImplementedError
283 def addjournal(self, classname, nodeid, action, params):
284 ''' Journal the Action
285 'action' may be:
287 'create' or 'set' -- 'params' is a dictionary of property values
288 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
289 'retire' -- 'params' is None
290 '''
291 raise NotImplementedError
293 def getjournal(self, classname, nodeid):
294 ''' get the journal for id
295 '''
296 raise NotImplementedError
298 def pack(self, pack_before):
299 ''' pack the database
300 '''
301 raise NotImplementedError
303 def commit(self):
304 ''' Commit the current transactions.
306 Save all data changed since the database was opened or since the
307 last commit() or rollback().
308 '''
309 raise NotImplementedError
311 def rollback(self):
312 ''' Reverse all actions from the current transaction.
314 Undo all the changes made since the database was opened or the last
315 commit() or rollback() was performed.
316 '''
317 raise NotImplementedError
319 #
320 # The base Class class
321 #
322 class Class:
323 """ The handle to a particular class of nodes in a hyperdatabase.
325 All methods except __repr__ and getnode must be implemented by a
326 concrete backend Class.
327 """
329 def __init__(self, db, classname, **properties):
330 """Create a new class with a given name and property specification.
332 'classname' must not collide with the name of an existing class,
333 or a ValueError is raised. The keyword arguments in 'properties'
334 must map names to property objects, or a TypeError is raised.
335 """
336 raise NotImplementedError
338 def __repr__(self):
339 '''Slightly more useful representation
340 '''
341 return '<hyperdb.Class "%s">'%self.classname
343 # Editing nodes:
345 def create(self, **propvalues):
346 """Create a new node of this class and return its id.
348 The keyword arguments in 'propvalues' map property names to values.
350 The values of arguments must be acceptable for the types of their
351 corresponding properties or a TypeError is raised.
353 If this class has a key property, it must be present and its value
354 must not collide with other key strings or a ValueError is raised.
356 Any other properties on this class that are missing from the
357 'propvalues' dictionary are set to None.
359 If an id in a link or multilink property does not refer to a valid
360 node, an IndexError is raised.
361 """
362 raise NotImplementedError
364 _marker = []
365 def get(self, nodeid, propname, default=_marker, cache=1):
366 """Get the value of a property on an existing node of this class.
368 'nodeid' must be the id of an existing node of this class or an
369 IndexError is raised. 'propname' must be the name of a property
370 of this class or a KeyError is raised.
372 'cache' exists for backwards compatibility, and is not used.
373 """
374 raise NotImplementedError
376 # not in spec
377 def getnode(self, nodeid):
378 ''' Return a convenience wrapper for the node.
380 'nodeid' must be the id of an existing node of this class or an
381 IndexError is raised.
383 'cache' exists for backwards compatibility, and is not used.
384 '''
385 return Node(self, nodeid)
387 def getnodeids(self, retired=None):
388 '''Retrieve all the ids of the nodes for a particular Class.
389 '''
390 raise NotImplementedError
392 def set(self, nodeid, **propvalues):
393 """Modify a property on an existing node of this class.
395 'nodeid' must be the id of an existing node of this class or an
396 IndexError is raised.
398 Each key in 'propvalues' must be the name of a property of this
399 class or a KeyError is raised.
401 All values in 'propvalues' must be acceptable types for their
402 corresponding properties or a TypeError is raised.
404 If the value of the key property is set, it must not collide with
405 other key strings or a ValueError is raised.
407 If the value of a Link or Multilink property contains an invalid
408 node id, a ValueError is raised.
409 """
410 raise NotImplementedError
412 def retire(self, nodeid):
413 """Retire a node.
415 The properties on the node remain available from the get() method,
416 and the node's id is never reused.
418 Retired nodes are not returned by the find(), list(), or lookup()
419 methods, and other nodes may reuse the values of their key properties.
420 """
421 raise NotImplementedError
423 def restore(self, nodeid):
424 '''Restpre a retired node.
426 Make node available for all operations like it was before retirement.
427 '''
428 raise NotImplementedError
430 def is_retired(self, nodeid):
431 '''Return true if the node is rerired
432 '''
433 raise NotImplementedError
435 def destroy(self, nodeid):
436 """Destroy a node.
438 WARNING: this method should never be used except in extremely rare
439 situations where there could never be links to the node being
440 deleted
442 WARNING: use retire() instead
444 WARNING: the properties of this node will not be available ever again
446 WARNING: really, use retire() instead
448 Well, I think that's enough warnings. This method exists mostly to
449 support the session storage of the cgi interface.
451 The node is completely removed from the hyperdb, including all journal
452 entries. It will no longer be available, and will generally break code
453 if there are any references to the node.
454 """
456 def history(self, nodeid):
457 """Retrieve the journal of edits on a particular node.
459 'nodeid' must be the id of an existing node of this class or an
460 IndexError is raised.
462 The returned list contains tuples of the form
464 (date, tag, action, params)
466 'date' is a Timestamp object specifying the time of the change and
467 'tag' is the journaltag specified when the database was opened.
468 """
469 raise NotImplementedError
471 # Locating nodes:
472 def hasnode(self, nodeid):
473 '''Determine if the given nodeid actually exists
474 '''
475 raise NotImplementedError
477 def setkey(self, propname):
478 """Select a String property of this class to be the key property.
480 'propname' must be the name of a String property of this class or
481 None, or a TypeError is raised. The values of the key property on
482 all existing nodes must be unique or a ValueError is raised.
483 """
484 raise NotImplementedError
486 def getkey(self):
487 """Return the name of the key property for this class or None."""
488 raise NotImplementedError
490 def labelprop(self, default_to_id=0):
491 """Return the property name for a label for the given node.
493 This method attempts to generate a consistent label for the node.
494 It tries the following in order:
496 1. key property
497 2. "name" property
498 3. "title" property
499 4. first property from the sorted property name list
500 """
501 raise NotImplementedError
503 def lookup(self, keyvalue):
504 """Locate a particular node by its key property and return its id.
506 If this class has no key property, a TypeError is raised. If the
507 'keyvalue' matches one of the values for the key property among
508 the nodes in this class, the matching node's id is returned;
509 otherwise a KeyError is raised.
510 """
511 raise NotImplementedError
513 def find(self, **propspec):
514 """Get the ids of nodes in this class which link to the given nodes.
516 'propspec' consists of keyword args propname={nodeid:1,}
517 'propname' must be the name of a property in this class, or a
518 KeyError is raised. That property must be a Link or Multilink
519 property, or a TypeError is raised.
521 Any node in this class whose 'propname' property links to any of the
522 nodeids will be returned. Used by the full text indexing, which knows
523 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
524 issues:
526 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
527 """
528 raise NotImplementedError
530 def filter(self, search_matches, filterspec, sort=(None,None),
531 group=(None,None)):
532 """Return a list of the ids of the active nodes in this class that
533 match the 'filter' spec, sorted by the group spec and then the
534 sort spec.
536 "filterspec" is {propname: value(s)}
538 "sort" and "group" are (dir, prop) where dir is '+', '-' or None
539 and prop is a prop name or None
541 "search_matches" is {nodeid: marker}
543 The filter must match all properties specificed - but if the
544 property value to match is a list, any one of the values in the
545 list may match for that property to match.
546 """
547 raise NotImplementedError
549 def count(self):
550 """Get the number of nodes in this class.
552 If the returned integer is 'numnodes', the ids of all the nodes
553 in this class run from 1 to numnodes, and numnodes+1 will be the
554 id of the next node to be created in this class.
555 """
556 raise NotImplementedError
558 # Manipulating properties:
559 def getprops(self, protected=1):
560 """Return a dictionary mapping property names to property objects.
561 If the "protected" flag is true, we include protected properties -
562 those which may not be modified.
563 """
564 raise NotImplementedError
566 def addprop(self, **properties):
567 """Add properties to this class.
569 The keyword arguments in 'properties' must map names to property
570 objects, or a TypeError is raised. None of the keys in 'properties'
571 may collide with the names of existing properties, or a ValueError
572 is raised before any properties have been added.
573 """
574 raise NotImplementedError
576 def index(self, nodeid):
577 '''Add (or refresh) the node to search indexes
578 '''
579 raise NotImplementedError
581 def safeget(self, nodeid, propname, default=None):
582 """Safely get the value of a property on an existing node of this class.
584 Return 'default' if the node doesn't exist.
585 """
586 try:
587 return self.get(nodeid, propname)
588 except IndexError:
589 return default
591 class HyperdbValueError(ValueError):
592 ''' Error converting a raw value into a Hyperdb value '''
593 pass
595 def convertLinkValue(db, propname, prop, value, idre=re.compile('\d+')):
596 ''' Convert the link value (may be id or key value) to an id value. '''
597 linkcl = db.classes[prop.classname]
598 if not idre.match(value):
599 if linkcl.getkey():
600 try:
601 value = linkcl.lookup(value)
602 except KeyError, message:
603 raise HyperdbValueError, 'property %s: %r is not a %s.'%(
604 propname, value, prop.classname)
605 else:
606 raise HyperdbValueError, 'you may only enter ID values '\
607 'for property %s'%propname
608 return value
610 def fixNewlines(text):
611 """ Homogenise line endings.
613 Different web clients send different line ending values, but
614 other systems (eg. email) don't necessarily handle those line
615 endings. Our solution is to convert all line endings to LF.
616 """
617 text = text.replace('\r\n', '\n')
618 return text.replace('\r', '\n')
620 def rawToHyperdb(db, klass, itemid, propname, value,
621 pwre=re.compile(r'{(\w+)}(.+)')):
622 ''' Convert the raw (user-input) value to a hyperdb-storable value. The
623 value is for the "propname" property on itemid (may be None for a
624 new item) of "klass" in "db".
626 The value is usually a string, but in the case of multilink inputs
627 it may be either a list of strings or a string with comma-separated
628 values.
629 '''
630 properties = klass.getprops()
632 # ensure it's a valid property name
633 propname = propname.strip()
634 try:
635 proptype = properties[propname]
636 except KeyError:
637 raise HyperdbValueError, '%r is not a property of %s'%(propname,
638 klass.classname)
640 # if we got a string, strip it now
641 if isinstance(value, type('')):
642 value = value.strip()
644 # convert the input value to a real property value
645 if isinstance(proptype, String):
646 # fix the CRLF/CR -> LF stuff
647 value = fixNewlines(value)
648 if isinstance(proptype, Password):
649 m = pwre.match(value)
650 if m:
651 # password is being given to us encrypted
652 p = password.Password()
653 p.scheme = m.group(1)
654 if p.scheme not in 'SHA crypt plaintext'.split():
655 raise HyperdbValueError, 'property %s: unknown encryption '\
656 'scheme %r'%(propname, p.scheme)
657 p.password = m.group(2)
658 value = p
659 else:
660 try:
661 value = password.Password(value)
662 except password.PasswordValueError, message:
663 raise HyperdbValueError, 'property %s: %s'%(propname, message)
664 elif isinstance(proptype, Date):
665 try:
666 tz = db.getUserTimezone()
667 value = date.Date(value).local(tz)
668 except ValueError, message:
669 raise HyperdbValueError, 'property %s: %r is an invalid '\
670 'date (%s)'%(propname, value, message)
671 elif isinstance(proptype, Interval):
672 try:
673 value = date.Interval(value)
674 except ValueError, message:
675 raise HyperdbValueError, 'property %s: %r is an invalid '\
676 'date interval (%s)'%(propname, value, message)
677 elif isinstance(proptype, Link):
678 if value == '-1' or not value:
679 value = None
680 else:
681 value = convertLinkValue(db, propname, proptype, value)
683 elif isinstance(proptype, Multilink):
684 # get the current item value if it's not a new item
685 if itemid and not itemid.startswith('-'):
686 curvalue = klass.get(itemid, propname)
687 else:
688 curvalue = []
690 # if the value is a comma-separated string then split it now
691 if isinstance(value, type('')):
692 value = value.split(',')
694 # handle each add/remove in turn
695 # keep an extra list for all items that are
696 # definitely in the new list (in case of e.g.
697 # <propname>=A,+B, which should replace the old
698 # list with A,B)
699 set = 1
700 newvalue = []
701 for item in value:
702 item = item.strip()
704 # skip blanks
705 if not item: continue
707 # handle +/-
708 remove = 0
709 if item.startswith('-'):
710 remove = 1
711 item = item[1:]
712 set = 0
713 elif item.startswith('+'):
714 item = item[1:]
715 set = 0
717 # look up the value
718 itemid = convertLinkValue(db, propname, proptype, item)
720 # perform the add/remove
721 if remove:
722 try:
723 curvalue.remove(itemid)
724 except ValueError:
725 raise HyperdbValueError, 'property %s: %r is not ' \
726 'currently an element'%(propname, item)
727 else:
728 newvalue.append(itemid)
729 if itemid not in curvalue:
730 curvalue.append(itemid)
732 # that's it, set the new Multilink property value,
733 # or overwrite it completely
734 if set:
735 value = newvalue
736 else:
737 value = curvalue
739 # TODO: one day, we'll switch to numeric ids and this will be
740 # unnecessary :(
741 value = [int(x) for x in value]
742 value.sort()
743 value = [str(x) for x in value]
744 elif isinstance(proptype, Boolean):
745 value = value.strip()
746 value = value.lower() in ('yes', 'true', 'on', '1')
747 elif isinstance(proptype, Number):
748 value = value.strip()
749 try:
750 value = float(value)
751 except ValueError:
752 raise HyperdbValueError, 'property %s: %r is not a number'%(
753 propname, value)
754 return value
756 class FileClass:
757 ''' A class that requires the "content" property and stores it on
758 disk.
759 '''
760 pass
762 class Node:
763 ''' A convenience wrapper for the given node
764 '''
765 def __init__(self, cl, nodeid, cache=1):
766 self.__dict__['cl'] = cl
767 self.__dict__['nodeid'] = nodeid
768 def keys(self, protected=1):
769 return self.cl.getprops(protected=protected).keys()
770 def values(self, protected=1):
771 l = []
772 for name in self.cl.getprops(protected=protected).keys():
773 l.append(self.cl.get(self.nodeid, name))
774 return l
775 def items(self, protected=1):
776 l = []
777 for name in self.cl.getprops(protected=protected).keys():
778 l.append((name, self.cl.get(self.nodeid, name)))
779 return l
780 def has_key(self, name):
781 return self.cl.getprops().has_key(name)
782 def get(self, name, default=None):
783 if self.has_key(name):
784 return self[name]
785 else:
786 return default
787 def __getattr__(self, name):
788 if self.__dict__.has_key(name):
789 return self.__dict__[name]
790 try:
791 return self.cl.get(self.nodeid, name)
792 except KeyError, value:
793 # we trap this but re-raise it as AttributeError - all other
794 # exceptions should pass through untrapped
795 pass
796 # nope, no such attribute
797 raise AttributeError, str(value)
798 def __getitem__(self, name):
799 return self.cl.get(self.nodeid, name)
800 def __setattr__(self, name, value):
801 try:
802 return self.cl.set(self.nodeid, **{name: value})
803 except KeyError, value:
804 raise AttributeError, str(value)
805 def __setitem__(self, name, value):
806 self.cl.set(self.nodeid, **{name: value})
807 def history(self):
808 return self.cl.history(self.nodeid)
809 def retire(self):
810 return self.cl.retire(self.nodeid)
813 def Choice(name, db, *options):
814 '''Quick helper to create a simple class with choices
815 '''
816 cl = Class(db, name, name=String(), order=String())
817 for i in range(len(options)):
818 cl.create(name=options[i], order=i)
819 return hyperdb.Link(name)
821 # vim: set filetype=python ts=4 sw=4 et si