1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 """Hyperdatabase implementation, especially field types.
20 """
21 __docformat__ = 'restructuredtext'
23 # standard python modules
24 import os, re, shutil, weakref
25 # Python 2.3 ... 2.6 compatibility:
26 from roundup.anypy.sets_ import set
28 # roundup modules
29 import date, password
30 from support import ensureParentsExist, PrioList, sorted, reversed
31 from roundup.i18n import _
33 #
34 # Types
35 #
36 class _Type(object):
37 """A roundup property type."""
38 def __init__(self, required=False):
39 self.required = required
40 def __repr__(self):
41 ' more useful for dumps '
42 return '<%s.%s>'%(self.__class__.__module__, self.__class__.__name__)
43 def sort_repr (self, cls, val, name):
44 """Representation used for sorting. This should be a python
45 built-in type, otherwise sorting will take ages. Note that
46 individual backends may chose to use something different for
47 sorting as long as the outcome is the same.
48 """
49 return val
51 class String(_Type):
52 """An object designating a String property."""
53 def __init__(self, indexme='no', required=False):
54 super(String, self).__init__(required)
55 self.indexme = indexme == 'yes'
56 def from_raw(self, value, propname='', **kw):
57 """fix the CRLF/CR -> LF stuff"""
58 if propname == 'content':
59 # Why oh why wasn't the FileClass content property a File
60 # type from the beginning?
61 return value
62 return fixNewlines(value)
63 def sort_repr (self, cls, val, name):
64 if not val:
65 return val
66 if name == 'id':
67 return int(val)
68 return val.lower()
70 class Password(_Type):
71 """An object designating a Password property."""
72 def from_raw(self, value, **kw):
73 if not value:
74 return None
75 m = password.Password.pwre.match(value)
76 if m:
77 # password is being given to us encrypted
78 p = password.Password()
79 p.scheme = m.group(1)
80 if p.scheme not in 'SHA crypt plaintext'.split():
81 raise HyperdbValueError, \
82 ('property %s: unknown encryption scheme %r') %\
83 (kw['propname'], p.scheme)
84 p.password = m.group(2)
85 value = p
86 else:
87 try:
88 value = password.Password(value)
89 except password.PasswordValueError, message:
90 raise HyperdbValueError, \
91 _('property %s: %s')%(kw['propname'], message)
92 return value
93 def sort_repr (self, cls, val, name):
94 if not val:
95 return val
96 return str(val)
98 class Date(_Type):
99 """An object designating a Date property."""
100 def __init__(self, offset=None, required=False):
101 super(Date, self).__init__(required)
102 self._offset = offset
103 def offset(self, db):
104 if self._offset is not None:
105 return self._offset
106 return db.getUserTimezone()
107 def from_raw(self, value, db, **kw):
108 try:
109 value = date.Date(value, self.offset(db))
110 except ValueError, message:
111 raise HyperdbValueError, _('property %s: %r is an invalid '\
112 'date (%s)')%(kw['propname'], value, message)
113 return value
114 def range_from_raw(self, value, db):
115 """return Range value from given raw value with offset correction"""
116 return date.Range(value, date.Date, offset=self.offset(db))
117 def sort_repr (self, cls, val, name):
118 if not val:
119 return val
120 return str(val)
122 class Interval(_Type):
123 """An object designating an Interval property."""
124 def from_raw(self, value, **kw):
125 try:
126 value = date.Interval(value)
127 except ValueError, message:
128 raise HyperdbValueError, _('property %s: %r is an invalid '\
129 'date interval (%s)')%(kw['propname'], value, message)
130 return value
131 def sort_repr (self, cls, val, name):
132 if not val:
133 return val
134 return val.as_seconds()
136 class _Pointer(_Type):
137 """An object designating a Pointer property that links or multilinks
138 to a node in a specified class."""
139 def __init__(self, classname, do_journal='yes', required=False):
140 """ Default is to journal link and unlink events
141 """
142 super(_Pointer, self).__init__(required)
143 self.classname = classname
144 self.do_journal = do_journal == 'yes'
145 def __repr__(self):
146 """more useful for dumps. But beware: This is also used in schema
147 storage in SQL backends!
148 """
149 return '<%s.%s to "%s">'%(self.__class__.__module__,
150 self.__class__.__name__, self.classname)
152 class Link(_Pointer):
153 """An object designating a Link property that links to a
154 node in a specified class."""
155 def from_raw(self, value, db, propname, **kw):
156 if value == '-1' or not value:
157 value = None
158 else:
159 value = convertLinkValue(db, propname, self, value)
160 return value
161 def sort_repr (self, cls, val, name):
162 if not val:
163 return val
164 op = cls.labelprop()
165 if op == 'id':
166 return int(cls.get(val, op))
167 return cls.get(val, op)
169 class Multilink(_Pointer):
170 """An object designating a Multilink property that links
171 to nodes in a specified class.
173 "classname" indicates the class to link to
175 "do_journal" indicates whether the linked-to nodes should have
176 'link' and 'unlink' events placed in their journal
177 """
178 def from_raw(self, value, db, klass, propname, itemid, **kw):
179 if not value:
180 return []
182 # get the current item value if it's not a new item
183 if itemid and not itemid.startswith('-'):
184 curvalue = klass.get(itemid, propname)
185 else:
186 curvalue = []
188 # if the value is a comma-separated string then split it now
189 if isinstance(value, type('')):
190 value = value.split(',')
192 # handle each add/remove in turn
193 # keep an extra list for all items that are
194 # definitely in the new list (in case of e.g.
195 # <propname>=A,+B, which should replace the old
196 # list with A,B)
197 do_set = 1
198 newvalue = []
199 for item in value:
200 item = item.strip()
202 # skip blanks
203 if not item: continue
205 # handle +/-
206 remove = 0
207 if item.startswith('-'):
208 remove = 1
209 item = item[1:]
210 do_set = 0
211 elif item.startswith('+'):
212 item = item[1:]
213 do_set = 0
215 # look up the value
216 itemid = convertLinkValue(db, propname, self, item)
218 # perform the add/remove
219 if remove:
220 try:
221 curvalue.remove(itemid)
222 except ValueError:
223 raise HyperdbValueError, _('property %s: %r is not ' \
224 'currently an element')%(propname, item)
225 else:
226 newvalue.append(itemid)
227 if itemid not in curvalue:
228 curvalue.append(itemid)
230 # that's it, set the new Multilink property value,
231 # or overwrite it completely
232 if do_set:
233 value = newvalue
234 else:
235 value = curvalue
237 # TODO: one day, we'll switch to numeric ids and this will be
238 # unnecessary :(
239 value = [int(x) for x in value]
240 value.sort()
241 value = [str(x) for x in value]
242 return value
244 def sort_repr (self, cls, val, name):
245 if not val:
246 return val
247 op = cls.labelprop()
248 if op == 'id':
249 return [int(cls.get(v, op)) for v in val]
250 return [cls.get(v, op) for v in val]
252 class Boolean(_Type):
253 """An object designating a boolean property"""
254 def from_raw(self, value, **kw):
255 value = value.strip()
256 # checked is a common HTML checkbox value
257 value = value.lower() in ('checked', 'yes', 'true', 'on', '1')
258 return value
260 class Number(_Type):
261 """An object designating a numeric property"""
262 def from_raw(self, value, **kw):
263 value = value.strip()
264 try:
265 value = float(value)
266 except ValueError:
267 raise HyperdbValueError, _('property %s: %r is not a number')%(
268 kw['propname'], value)
269 return value
270 #
271 # Support for splitting designators
272 #
273 class DesignatorError(ValueError):
274 pass
275 def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')):
276 """ Take a foo123 and return ('foo', 123)
277 """
278 m = dre.match(designator)
279 if m is None:
280 raise DesignatorError, _('"%s" not a node designator')%designator
281 return m.group(1), m.group(2)
283 class Proptree(object):
284 """ Simple tree data structure for optimizing searching of
285 properties. Each node in the tree represents a roundup Class
286 Property that has to be navigated for finding the given search
287 or sort properties. The sort_type attribute is used for
288 distinguishing nodes in the tree used for sorting or searching: If
289 it is 0 for a node, that node is not used for sorting. If it is 1,
290 it is used for both, sorting and searching. If it is 2 it is used
291 for sorting only.
293 The Proptree is also used for transitively searching attributes for
294 backends that do not support transitive search (e.g. anydbm). The
295 _val attribute with set_val is used for this.
296 """
298 def __init__(self, db, cls, name, props, parent = None):
299 self.db = db
300 self.name = name
301 self.props = props
302 self.parent = parent
303 self._val = None
304 self.has_values = False
305 self.cls = cls
306 self.classname = None
307 self.uniqname = None
308 self.children = []
309 self.sortattr = []
310 self.propdict = {}
311 self.sort_type = 0
312 self.sort_direction = None
313 self.sort_ids = None
314 self.sort_ids_needed = False
315 self.sort_result = None
316 self.attr_sort_done = False
317 self.tree_sort_done = False
318 self.propclass = None
319 self.orderby = []
320 if parent:
321 self.root = parent.root
322 self.depth = parent.depth + 1
323 else:
324 self.root = self
325 self.seqno = 1
326 self.depth = 0
327 self.sort_type = 1
328 self.id = self.root.seqno
329 self.root.seqno += 1
330 if self.cls:
331 self.classname = self.cls.classname
332 self.uniqname = '%s%s' % (self.cls.classname, self.id)
333 if not self.parent:
334 self.uniqname = self.cls.classname
336 def append(self, name, sort_type = 0):
337 """Append a property to self.children. Will create a new
338 propclass for the child.
339 """
340 if name in self.propdict:
341 pt = self.propdict[name]
342 if sort_type and not pt.sort_type:
343 pt.sort_type = 1
344 return pt
345 propclass = self.props[name]
346 cls = None
347 props = None
348 if isinstance(propclass, (Link, Multilink)):
349 cls = self.db.getclass(propclass.classname)
350 props = cls.getprops()
351 child = self.__class__(self.db, cls, name, props, parent = self)
352 child.sort_type = sort_type
353 child.propclass = propclass
354 self.children.append(child)
355 self.propdict[name] = child
356 return child
358 def compute_sort_done(self, mlseen=False):
359 """ Recursively check if attribute is needed for sorting
360 (self.sort_type > 0) or all children have tree_sort_done set and
361 sort_ids_needed unset: set self.tree_sort_done if one of the conditions
362 holds. Also remove sort_ids_needed recursively once having seen a
363 Multilink.
364 """
365 if isinstance (self.propclass, Multilink):
366 mlseen = True
367 if mlseen:
368 self.sort_ids_needed = False
369 self.tree_sort_done = True
370 for p in self.children:
371 p.compute_sort_done(mlseen)
372 if not p.tree_sort_done:
373 self.tree_sort_done = False
374 if not self.sort_type:
375 self.tree_sort_done = True
376 if mlseen:
377 self.tree_sort_done = False
379 def ancestors(self):
380 p = self
381 while p.parent:
382 yield p
383 p = p.parent
385 def search(self, search_matches=None, sort=True):
386 """ Recursively search for the given properties in a proptree.
387 Once all properties are non-transitive, the search generates a
388 simple _filter call which does the real work
389 """
390 filterspec = {}
391 for p in self.children:
392 if p.sort_type < 2:
393 if p.children:
394 p.search(sort = False)
395 filterspec[p.name] = p.val
396 self.val = self.cls._filter(search_matches, filterspec, sort and self)
397 return self.val
399 def sort (self, ids=None):
400 """ Sort ids by the order information stored in self. With
401 optimisations: Some order attributes may be precomputed (by the
402 backend) and some properties may already be sorted.
403 """
404 if ids is None:
405 ids = self.val
406 if self.sortattr and [s for s in self.sortattr if not s.attr_sort_done]:
407 return self._searchsort(ids, True, True)
408 return ids
410 def sortable_children(self, intermediate=False):
411 """ All children needed for sorting. If intermediate is True,
412 intermediate nodes (not being a sort attribute) are returned,
413 too.
414 """
415 return [p for p in self.children
416 if p.sort_type > 0 and (intermediate or p.sort_direction)]
418 def __iter__(self):
419 """ Yield nodes in depth-first order -- visited nodes first """
420 for p in self.children:
421 yield p
422 for c in p:
423 yield c
425 def _get (self, ids):
426 """Lookup given ids -- possibly a list of list. We recurse until
427 we have a list of ids.
428 """
429 if not ids:
430 return ids
431 if isinstance (ids[0], list):
432 cids = [self._get(i) for i in ids]
433 else:
434 cids = [i and self.parent.cls.get(i, self.name) for i in ids]
435 if self.sortattr:
436 cids = [self._searchsort(i, False, True) for i in cids]
437 return cids
439 def _searchsort(self, ids=None, update=True, dosort=True):
440 """ Recursively compute the sort attributes. Note that ids
441 may be a deeply nested list of lists of ids if several
442 multilinks are encountered on the way from the root to an
443 individual attribute. We make sure that everything is properly
444 sorted on the way up. Note that the individual backend may
445 already have precomputed self.result or self.sort_ids. In this
446 case we do nothing for existing sa.result and recurse further if
447 self.sort_ids is available.
449 Yech, Multilinks: This gets especially complicated if somebody
450 sorts by different attributes of the same multilink (or
451 transitively across several multilinks). My use-case is sorting
452 by issue.messages.author and (reverse) by issue.messages.date.
453 In this case we sort the messages by author and date and use
454 this sorted list twice for sorting issues. This means that
455 issues are sorted by author and then by the time of the messages
456 *of this author*. Probably what the user intends in that case,
457 so we do *not* use two sorted lists of messages, one sorted by
458 author and one sorted by date for sorting issues.
459 """
460 for pt in self.sortable_children(intermediate = True):
461 # ids can be an empty list
462 if pt.tree_sort_done or not ids:
463 continue
464 if pt.sort_ids: # cached or computed by backend
465 cids = pt.sort_ids
466 else:
467 cids = pt._get(ids)
468 if pt.sort_direction and not pt.sort_result:
469 sortrep = pt.propclass.sort_repr
470 pt.sort_result = pt._sort_repr(sortrep, cids)
471 pt.sort_ids = cids
472 if pt.children:
473 pt._searchsort(cids, update, False)
474 if self.sortattr and dosort:
475 ids = self._sort(ids)
476 if not update:
477 for pt in self.sortable_children(intermediate = True):
478 pt.sort_ids = None
479 for pt in self.sortattr:
480 pt.sort_result = None
481 return ids
483 def _set_val(self, val):
484 """Check if self._val is already defined. If yes, we compute the
485 intersection of the old and the new value(s)
486 """
487 if self.has_values:
488 v = self._val
489 if not isinstance(self._val, type([])):
490 v = [self._val]
491 vals = set(v)
492 vals.intersection_update(val)
493 self._val = [v for v in vals]
494 else:
495 self._val = val
496 self.has_values = True
498 val = property(lambda self: self._val, _set_val)
500 def _sort(self, val):
501 """Finally sort by the given sortattr.sort_result. Note that we
502 do not sort by attrs having attr_sort_done set. The caller is
503 responsible for setting attr_sort_done only for trailing
504 attributes (otherwise the sort order is wrong). Since pythons
505 sort is stable, we can sort already sorted lists without
506 destroying the sort-order for items that compare equal with the
507 current sort.
509 Sorting-Strategy: We sort repeatedly by different sort-keys from
510 right to left. Since pythons sort is stable, we can safely do
511 that. An optimisation is a "run-length encoding" of the
512 sort-directions: If several sort attributes sort in the same
513 direction we can combine them into a single sort. Note that
514 repeated sorting is probably more efficient than using
515 compare-methods in python due to the overhead added by compare
516 methods.
517 """
518 if not val:
519 return val
520 sortattr = []
521 directions = []
522 dir_idx = []
523 idx = 0
524 curdir = None
525 for sa in self.sortattr:
526 if sa.attr_sort_done:
527 break
528 if sortattr:
529 assert len(sortattr[0]) == len(sa.sort_result)
530 sortattr.append (sa.sort_result)
531 if curdir != sa.sort_direction:
532 dir_idx.append (idx)
533 directions.append (sa.sort_direction)
534 curdir = sa.sort_direction
535 idx += 1
536 sortattr.append (val)
537 #print >> sys.stderr, "\nsortattr", sortattr
538 sortattr = zip (*sortattr)
539 for dir, i in reversed(zip(directions, dir_idx)):
540 rev = dir == '-'
541 sortattr = sorted (sortattr, key = lambda x:x[i:idx], reverse = rev)
542 idx = i
543 return [x[-1] for x in sortattr]
545 def _sort_repr(self, sortrep, ids):
546 """Call sortrep for given ids -- possibly a list of list. We
547 recurse until we have a list of ids.
548 """
549 if not ids:
550 return ids
551 if isinstance (ids[0], list):
552 res = [self._sort_repr(sortrep, i) for i in ids]
553 else:
554 res = [sortrep(self.cls, i, self.name) for i in ids]
555 return res
557 def __repr__(self):
558 r = ["proptree:" + self.name]
559 for n in self:
560 r.append("proptree:" + " " * n.depth + n.name)
561 return '\n'.join(r)
562 __str__ = __repr__
564 #
565 # the base Database class
566 #
567 class DatabaseError(ValueError):
568 """Error to be raised when there is some problem in the database code
569 """
570 pass
571 class Database:
572 """A database for storing records containing flexible data types.
574 This class defines a hyperdatabase storage layer, which the Classes use to
575 store their data.
578 Transactions
579 ------------
580 The Database should support transactions through the commit() and
581 rollback() methods. All other Database methods should be transaction-aware,
582 using data from the current transaction before looking up the database.
584 An implementation must provide an override for the get() method so that the
585 in-database value is returned in preference to the in-transaction value.
586 This is necessary to determine if any values have changed during a
587 transaction.
590 Implementation
591 --------------
593 All methods except __repr__ must be implemented by a concrete backend Database.
595 """
597 # flag to set on retired entries
598 RETIRED_FLAG = '__hyperdb_retired'
600 BACKEND_MISSING_STRING = None
601 BACKEND_MISSING_NUMBER = None
602 BACKEND_MISSING_BOOLEAN = None
604 def __init__(self, config, journaltag=None):
605 """Open a hyperdatabase given a specifier to some storage.
607 The 'storagelocator' is obtained from config.DATABASE.
608 The meaning of 'storagelocator' depends on the particular
609 implementation of the hyperdatabase. It could be a file name,
610 a directory path, a socket descriptor for a connection to a
611 database over the network, etc.
613 The 'journaltag' is a token that will be attached to the journal
614 entries for any edits done on the database. If 'journaltag' is
615 None, the database is opened in read-only mode: the Class.create(),
616 Class.set(), and Class.retire() methods are disabled.
617 """
618 raise NotImplementedError
620 def post_init(self):
621 """Called once the schema initialisation has finished.
622 If 'refresh' is true, we want to rebuild the backend
623 structures.
624 """
625 raise NotImplementedError
627 def refresh_database(self):
628 """Called to indicate that the backend should rebuild all tables
629 and structures. Not called in normal usage."""
630 raise NotImplementedError
632 def __getattr__(self, classname):
633 """A convenient way of calling self.getclass(classname)."""
634 raise NotImplementedError
636 def addclass(self, cl):
637 """Add a Class to the hyperdatabase.
638 """
639 raise NotImplementedError
641 def getclasses(self):
642 """Return a list of the names of all existing classes."""
643 raise NotImplementedError
645 def getclass(self, classname):
646 """Get the Class object representing a particular class.
648 If 'classname' is not a valid class name, a KeyError is raised.
649 """
650 raise NotImplementedError
652 def clear(self):
653 """Delete all database contents.
654 """
655 raise NotImplementedError
657 def getclassdb(self, classname, mode='r'):
658 """Obtain a connection to the class db that will be used for
659 multiple actions.
660 """
661 raise NotImplementedError
663 def addnode(self, classname, nodeid, node):
664 """Add the specified node to its class's db.
665 """
666 raise NotImplementedError
668 def serialise(self, classname, node):
669 """Copy the node contents, converting non-marshallable data into
670 marshallable data.
671 """
672 return node
674 def setnode(self, classname, nodeid, node):
675 """Change the specified node.
676 """
677 raise NotImplementedError
679 def unserialise(self, classname, node):
680 """Decode the marshalled node data
681 """
682 return node
684 def getnode(self, classname, nodeid):
685 """Get a node from the database.
687 'cache' exists for backwards compatibility, and is not used.
688 """
689 raise NotImplementedError
691 def hasnode(self, classname, nodeid):
692 """Determine if the database has a given node.
693 """
694 raise NotImplementedError
696 def countnodes(self, classname):
697 """Count the number of nodes that exist for a particular Class.
698 """
699 raise NotImplementedError
701 def storefile(self, classname, nodeid, property, content):
702 """Store the content of the file in the database.
704 The property may be None, in which case the filename does not
705 indicate which property is being saved.
706 """
707 raise NotImplementedError
709 def getfile(self, classname, nodeid, property):
710 """Get the content of the file in the database.
711 """
712 raise NotImplementedError
714 def addjournal(self, classname, nodeid, action, params):
715 """ Journal the Action
716 'action' may be:
718 'create' or 'set' -- 'params' is a dictionary of property values
719 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
720 'retire' -- 'params' is None
721 """
722 raise NotImplementedError
724 def getjournal(self, classname, nodeid):
725 """ get the journal for id
726 """
727 raise NotImplementedError
729 def pack(self, pack_before):
730 """ pack the database
731 """
732 raise NotImplementedError
734 def commit(self):
735 """ Commit the current transactions.
737 Save all data changed since the database was opened or since the
738 last commit() or rollback().
740 fail_ok indicates that the commit is allowed to fail. This is used
741 in the web interface when committing cleaning of the session
742 database. We don't care if there's a concurrency issue there.
744 The only backend this seems to affect is postgres.
745 """
746 raise NotImplementedError
748 def rollback(self):
749 """ Reverse all actions from the current transaction.
751 Undo all the changes made since the database was opened or the last
752 commit() or rollback() was performed.
753 """
754 raise NotImplementedError
756 def close(self):
757 """Close the database.
759 This method must be called at the end of processing.
761 """
763 def iter_roles(roles):
764 ''' handle the text processing of turning the roles list
765 into something python can use more easily
766 '''
767 if not roles or not roles.strip():
768 raise StopIteration, "Empty roles given"
769 for role in [x.lower().strip() for x in roles.split(',')]:
770 yield role
773 #
774 # The base Class class
775 #
776 class Class:
777 """ The handle to a particular class of nodes in a hyperdatabase.
779 All methods except __repr__ and getnode must be implemented by a
780 concrete backend Class.
781 """
783 def __init__(self, db, classname, **properties):
784 """Create a new class with a given name and property specification.
786 'classname' must not collide with the name of an existing class,
787 or a ValueError is raised. The keyword arguments in 'properties'
788 must map names to property objects, or a TypeError is raised.
789 """
790 for name in 'creation activity creator actor'.split():
791 if properties.has_key(name):
792 raise ValueError, '"creation", "activity", "creator" and '\
793 '"actor" are reserved'
795 self.classname = classname
796 self.properties = properties
797 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
798 self.key = ''
800 # should we journal changes (default yes)
801 self.do_journal = 1
803 # do the db-related init stuff
804 db.addclass(self)
806 actions = "create set retire restore".split()
807 self.auditors = dict([(a, PrioList()) for a in actions])
808 self.reactors = dict([(a, PrioList()) for a in actions])
810 def __repr__(self):
811 """Slightly more useful representation
812 """
813 return '<hyperdb.Class "%s">'%self.classname
815 # Editing nodes:
817 def create(self, **propvalues):
818 """Create a new node of this class and return its id.
820 The keyword arguments in 'propvalues' map property names to values.
822 The values of arguments must be acceptable for the types of their
823 corresponding properties or a TypeError is raised.
825 If this class has a key property, it must be present and its value
826 must not collide with other key strings or a ValueError is raised.
828 Any other properties on this class that are missing from the
829 'propvalues' dictionary are set to None.
831 If an id in a link or multilink property does not refer to a valid
832 node, an IndexError is raised.
833 """
834 raise NotImplementedError
836 _marker = []
837 def get(self, nodeid, propname, default=_marker, cache=1):
838 """Get the value of a property on an existing node of this class.
840 'nodeid' must be the id of an existing node of this class or an
841 IndexError is raised. 'propname' must be the name of a property
842 of this class or a KeyError is raised.
844 'cache' exists for backwards compatibility, and is not used.
845 """
846 raise NotImplementedError
848 # not in spec
849 def getnode(self, nodeid):
850 """ Return a convenience wrapper for the node.
852 'nodeid' must be the id of an existing node of this class or an
853 IndexError is raised.
855 'cache' exists for backwards compatibility, and is not used.
856 """
857 return Node(self, nodeid)
859 def getnodeids(self, retired=None):
860 """Retrieve all the ids of the nodes for a particular Class.
861 """
862 raise NotImplementedError
864 def set(self, nodeid, **propvalues):
865 """Modify a property on an existing node of this class.
867 'nodeid' must be the id of an existing node of this class or an
868 IndexError is raised.
870 Each key in 'propvalues' must be the name of a property of this
871 class or a KeyError is raised.
873 All values in 'propvalues' must be acceptable types for their
874 corresponding properties or a TypeError is raised.
876 If the value of the key property is set, it must not collide with
877 other key strings or a ValueError is raised.
879 If the value of a Link or Multilink property contains an invalid
880 node id, a ValueError is raised.
881 """
882 raise NotImplementedError
884 def retire(self, nodeid):
885 """Retire a node.
887 The properties on the node remain available from the get() method,
888 and the node's id is never reused.
890 Retired nodes are not returned by the find(), list(), or lookup()
891 methods, and other nodes may reuse the values of their key properties.
892 """
893 raise NotImplementedError
895 def restore(self, nodeid):
896 """Restpre a retired node.
898 Make node available for all operations like it was before retirement.
899 """
900 raise NotImplementedError
902 def is_retired(self, nodeid):
903 """Return true if the node is rerired
904 """
905 raise NotImplementedError
907 def destroy(self, nodeid):
908 """Destroy a node.
910 WARNING: this method should never be used except in extremely rare
911 situations where there could never be links to the node being
912 deleted
914 WARNING: use retire() instead
916 WARNING: the properties of this node will not be available ever again
918 WARNING: really, use retire() instead
920 Well, I think that's enough warnings. This method exists mostly to
921 support the session storage of the cgi interface.
923 The node is completely removed from the hyperdb, including all journal
924 entries. It will no longer be available, and will generally break code
925 if there are any references to the node.
926 """
928 def history(self, nodeid):
929 """Retrieve the journal of edits on a particular node.
931 'nodeid' must be the id of an existing node of this class or an
932 IndexError is raised.
934 The returned list contains tuples of the form
936 (date, tag, action, params)
938 'date' is a Timestamp object specifying the time of the change and
939 'tag' is the journaltag specified when the database was opened.
940 """
941 raise NotImplementedError
943 # Locating nodes:
944 def hasnode(self, nodeid):
945 """Determine if the given nodeid actually exists
946 """
947 raise NotImplementedError
949 def setkey(self, propname):
950 """Select a String property of this class to be the key property.
952 'propname' must be the name of a String property of this class or
953 None, or a TypeError is raised. The values of the key property on
954 all existing nodes must be unique or a ValueError is raised.
955 """
956 raise NotImplementedError
958 def setlabelprop(self, labelprop):
959 """Set the label property. Used for override of labelprop
960 resolution order.
961 """
962 if labelprop not in self.getprops():
963 raise ValueError, _("Not a property name: %s") % labelprop
964 self._labelprop = labelprop
966 def setorderprop(self, orderprop):
967 """Set the order property. Used for override of orderprop
968 resolution order
969 """
970 if orderprop not in self.getprops():
971 raise ValueError, _("Not a property name: %s") % orderprop
972 self._orderprop = orderprop
974 def getkey(self):
975 """Return the name of the key property for this class or None."""
976 raise NotImplementedError
978 def labelprop(self, default_to_id=0):
979 """Return the property name for a label for the given node.
981 This method attempts to generate a consistent label for the node.
982 It tries the following in order:
984 0. self._labelprop if set
985 1. key property
986 2. "name" property
987 3. "title" property
988 4. first property from the sorted property name list
989 """
990 if hasattr(self, '_labelprop'):
991 return self._labelprop
992 k = self.getkey()
993 if k:
994 return k
995 props = self.getprops()
996 if props.has_key('name'):
997 return 'name'
998 elif props.has_key('title'):
999 return 'title'
1000 if default_to_id:
1001 return 'id'
1002 props = props.keys()
1003 props.sort()
1004 return props[0]
1006 def orderprop(self):
1007 """Return the property name to use for sorting for the given node.
1009 This method computes the property for sorting.
1010 It tries the following in order:
1012 0. self._orderprop if set
1013 1. "order" property
1014 2. self.labelprop()
1015 """
1017 if hasattr(self, '_orderprop'):
1018 return self._orderprop
1019 props = self.getprops()
1020 if props.has_key('order'):
1021 return 'order'
1022 return self.labelprop()
1024 def lookup(self, keyvalue):
1025 """Locate a particular node by its key property and return its id.
1027 If this class has no key property, a TypeError is raised. If the
1028 'keyvalue' matches one of the values for the key property among
1029 the nodes in this class, the matching node's id is returned;
1030 otherwise a KeyError is raised.
1031 """
1032 raise NotImplementedError
1034 def find(self, **propspec):
1035 """Get the ids of nodes in this class which link to the given nodes.
1037 'propspec' consists of keyword args propname={nodeid:1,}
1038 'propname' must be the name of a property in this class, or a
1039 KeyError is raised. That property must be a Link or Multilink
1040 property, or a TypeError is raised.
1042 Any node in this class whose 'propname' property links to any of the
1043 nodeids will be returned. Used by the full text indexing, which knows
1044 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1045 issues:
1047 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1048 """
1049 raise NotImplementedError
1051 def _filter(self, search_matches, filterspec, sort=(None,None),
1052 group=(None,None)):
1053 """For some backends this implements the non-transitive
1054 search, for more information see the filter method.
1055 """
1056 raise NotImplementedError
1058 def _proptree(self, filterspec, sortattr=[]):
1059 """Build a tree of all transitive properties in the given
1060 filterspec.
1061 """
1062 proptree = Proptree(self.db, self, '', self.getprops())
1063 for key, v in filterspec.iteritems():
1064 keys = key.split('.')
1065 p = proptree
1066 for k in keys:
1067 p = p.append(k)
1068 p.val = v
1069 multilinks = {}
1070 for s in sortattr:
1071 keys = s[1].split('.')
1072 p = proptree
1073 for k in keys:
1074 p = p.append(k, sort_type = 2)
1075 if isinstance (p.propclass, Multilink):
1076 multilinks[p] = True
1077 if p.cls:
1078 p = p.append(p.cls.orderprop(), sort_type = 2)
1079 if p.sort_direction: # if an orderprop is also specified explicitly
1080 continue
1081 p.sort_direction = s[0]
1082 proptree.sortattr.append (p)
1083 for p in multilinks.iterkeys():
1084 sattr = {}
1085 for c in p:
1086 if c.sort_direction:
1087 sattr [c] = True
1088 for sa in proptree.sortattr:
1089 if sa in sattr:
1090 p.sortattr.append (sa)
1091 return proptree
1093 def get_transitive_prop(self, propname_path, default = None):
1094 """Expand a transitive property (individual property names
1095 separated by '.' into a new property at the end of the path. If
1096 one of the names does not refer to a valid property, we return
1097 None.
1098 Example propname_path (for class issue): "messages.author"
1099 """
1100 props = self.db.getclass(self.classname).getprops()
1101 for k in propname_path.split('.'):
1102 try:
1103 prop = props[k]
1104 except KeyError, TypeError:
1105 return default
1106 cl = getattr(prop, 'classname', None)
1107 props = None
1108 if cl:
1109 props = self.db.getclass(cl).getprops()
1110 return prop
1112 def _sortattr(self, sort=[], group=[]):
1113 """Build a single list of sort attributes in the correct order
1114 with sanity checks (no duplicate properties) included. Always
1115 sort last by id -- if id is not already in sortattr.
1116 """
1117 seen = {}
1118 sortattr = []
1119 for srt in group, sort:
1120 if not isinstance(srt, list):
1121 srt = [srt]
1122 for s in srt:
1123 if s[1] and s[1] not in seen:
1124 sortattr.append((s[0] or '+', s[1]))
1125 seen[s[1]] = True
1126 if 'id' not in seen :
1127 sortattr.append(('+', 'id'))
1128 return sortattr
1130 def filter(self, search_matches, filterspec, sort=[], group=[]):
1131 """Return a list of the ids of the active nodes in this class that
1132 match the 'filter' spec, sorted by the group spec and then the
1133 sort spec.
1135 "filterspec" is {propname: value(s)}
1137 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
1138 or None and prop is a prop name or None. Note that for
1139 backward-compatibility reasons a single (dir, prop) tuple is
1140 also allowed.
1142 "search_matches" is a container type
1144 The filter must match all properties specificed. If the property
1145 value to match is a list:
1147 1. String properties must match all elements in the list, and
1148 2. Other properties must match any of the elements in the list.
1150 Note that now the propname in filterspec and prop in a
1151 sort/group spec may be transitive, i.e., it may contain
1152 properties of the form link.link.link.name, e.g. you can search
1153 for all issues where a message was added by a certain user in
1154 the last week with a filterspec of
1155 {'messages.author' : '42', 'messages.creation' : '.-1w;'}
1157 Implementation note:
1158 This implements a non-optimized version of Transitive search
1159 using _filter implemented in a backend class. A more efficient
1160 version can be implemented in the individual backends -- e.g.,
1161 an SQL backen will want to create a single SQL statement and
1162 override the filter method instead of implementing _filter.
1163 """
1164 sortattr = self._sortattr(sort = sort, group = group)
1165 proptree = self._proptree(filterspec, sortattr)
1166 proptree.search(search_matches)
1167 return proptree.sort()
1169 def count(self):
1170 """Get the number of nodes in this class.
1172 If the returned integer is 'numnodes', the ids of all the nodes
1173 in this class run from 1 to numnodes, and numnodes+1 will be the
1174 id of the next node to be created in this class.
1175 """
1176 raise NotImplementedError
1178 # Manipulating properties:
1179 def getprops(self, protected=1):
1180 """Return a dictionary mapping property names to property objects.
1181 If the "protected" flag is true, we include protected properties -
1182 those which may not be modified.
1183 """
1184 raise NotImplementedError
1186 def get_required_props(self, propnames = []):
1187 """Return a dict of property names mapping to property objects.
1188 All properties that have the "required" flag set will be
1189 returned in addition to all properties in the propnames
1190 parameter.
1191 """
1192 props = self.getprops(protected = False)
1193 pdict = dict([(p, props[p]) for p in propnames])
1194 pdict.update([(k, v) for k, v in props.iteritems() if v.required])
1195 return pdict
1197 def addprop(self, **properties):
1198 """Add properties to this class.
1200 The keyword arguments in 'properties' must map names to property
1201 objects, or a TypeError is raised. None of the keys in 'properties'
1202 may collide with the names of existing properties, or a ValueError
1203 is raised before any properties have been added.
1204 """
1205 raise NotImplementedError
1207 def index(self, nodeid):
1208 """Add (or refresh) the node to search indexes"""
1209 raise NotImplementedError
1211 #
1212 # Detector interface
1213 #
1214 def audit(self, event, detector, priority = 100):
1215 """Register an auditor detector"""
1216 self.auditors[event].append((priority, detector.__name__, detector))
1218 def fireAuditors(self, event, nodeid, newvalues):
1219 """Fire all registered auditors"""
1220 for prio, name, audit in self.auditors[event]:
1221 audit(self.db, self, nodeid, newvalues)
1223 def react(self, event, detector, priority = 100):
1224 """Register a reactor detector"""
1225 self.reactors[event].append((priority, detector.__name__, detector))
1227 def fireReactors(self, event, nodeid, oldvalues):
1228 """Fire all registered reactors"""
1229 for prio, name, react in self.reactors[event]:
1230 react(self.db, self, nodeid, oldvalues)
1232 #
1233 # import / export support
1234 #
1235 def export_propnames(self):
1236 """List the property names for export from this Class"""
1237 propnames = self.getprops().keys()
1238 propnames.sort()
1239 return propnames
1240 #
1241 # convenience methods
1242 #
1243 def get_roles(self, nodeid):
1244 """Return iterator for all roles for this nodeid.
1246 Yields string-processed roles.
1247 This method can be overridden to provide a hook where we can
1248 insert other permission models (e.g. get roles from database)
1249 In standard schemas only a user has a roles property but
1250 this may be different in customized schemas.
1251 Note that this is the *central place* where role
1252 processing happens!
1253 """
1254 node = self.db.getnode(self.classname, nodeid)
1255 return iter_roles(node['roles'])
1257 def has_role(self, nodeid, *roles):
1258 '''See if this node has any roles that appear in roles.
1260 For convenience reasons we take a list.
1261 In standard schemas only a user has a roles property but
1262 this may be different in customized schemas.
1263 '''
1264 roles = dict.fromkeys ([r.strip().lower() for r in roles])
1265 for role in self.get_roles(nodeid):
1266 if role in roles:
1267 return True
1268 return False
1271 class HyperdbValueError(ValueError):
1272 """ Error converting a raw value into a Hyperdb value """
1273 pass
1275 def convertLinkValue(db, propname, prop, value, idre=re.compile('^\d+$')):
1276 """ Convert the link value (may be id or key value) to an id value. """
1277 linkcl = db.classes[prop.classname]
1278 if not idre.match(value):
1279 if linkcl.getkey():
1280 try:
1281 value = linkcl.lookup(value)
1282 except KeyError, message:
1283 raise HyperdbValueError, _('property %s: %r is not a %s.')%(
1284 propname, value, prop.classname)
1285 else:
1286 raise HyperdbValueError, _('you may only enter ID values '\
1287 'for property %s')%propname
1288 return value
1290 def fixNewlines(text):
1291 """ Homogenise line endings.
1293 Different web clients send different line ending values, but
1294 other systems (eg. email) don't necessarily handle those line
1295 endings. Our solution is to convert all line endings to LF.
1296 """
1297 text = text.replace('\r\n', '\n')
1298 return text.replace('\r', '\n')
1300 def rawToHyperdb(db, klass, itemid, propname, value, **kw):
1301 """ Convert the raw (user-input) value to a hyperdb-storable value. The
1302 value is for the "propname" property on itemid (may be None for a
1303 new item) of "klass" in "db".
1305 The value is usually a string, but in the case of multilink inputs
1306 it may be either a list of strings or a string with comma-separated
1307 values.
1308 """
1309 properties = klass.getprops()
1311 # ensure it's a valid property name
1312 propname = propname.strip()
1313 try:
1314 proptype = properties[propname]
1315 except KeyError:
1316 raise HyperdbValueError, _('%r is not a property of %s')%(propname,
1317 klass.classname)
1319 # if we got a string, strip it now
1320 if isinstance(value, type('')):
1321 value = value.strip()
1323 # convert the input value to a real property value
1324 value = proptype.from_raw(value, db=db, klass=klass,
1325 propname=propname, itemid=itemid, **kw)
1327 return value
1329 class FileClass:
1330 """ A class that requires the "content" property and stores it on
1331 disk.
1332 """
1333 default_mime_type = 'text/plain'
1335 def __init__(self, db, classname, **properties):
1336 """The newly-created class automatically includes the "content"
1337 property.
1338 """
1339 if not properties.has_key('content'):
1340 properties['content'] = String(indexme='yes')
1342 def export_propnames(self):
1343 """ Don't export the "content" property
1344 """
1345 propnames = self.getprops().keys()
1346 propnames.remove('content')
1347 propnames.sort()
1348 return propnames
1350 def exportFilename(self, dirname, nodeid):
1351 subdir_filename = self.db.subdirFilename(self.classname, nodeid)
1352 return os.path.join(dirname, self.classname+'-files', subdir_filename)
1354 def export_files(self, dirname, nodeid):
1355 """ Export the "content" property as a file, not csv column
1356 """
1357 source = self.db.filename(self.classname, nodeid)
1359 dest = self.exportFilename(dirname, nodeid)
1360 ensureParentsExist(dest)
1361 shutil.copyfile(source, dest)
1363 def import_files(self, dirname, nodeid):
1364 """ Import the "content" property as a file
1365 """
1366 source = self.exportFilename(dirname, nodeid)
1368 dest = self.db.filename(self.classname, nodeid, create=1)
1369 ensureParentsExist(dest)
1370 shutil.copyfile(source, dest)
1372 mime_type = None
1373 props = self.getprops()
1374 if props.has_key('type'):
1375 mime_type = self.get(nodeid, 'type')
1376 if not mime_type:
1377 mime_type = self.default_mime_type
1378 if props['content'].indexme:
1379 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1380 self.get(nodeid, 'content'), mime_type)
1382 class Node:
1383 """ A convenience wrapper for the given node
1384 """
1385 def __init__(self, cl, nodeid, cache=1):
1386 self.__dict__['cl'] = cl
1387 self.__dict__['nodeid'] = nodeid
1388 def keys(self, protected=1):
1389 return self.cl.getprops(protected=protected).keys()
1390 def values(self, protected=1):
1391 l = []
1392 for name in self.cl.getprops(protected=protected).keys():
1393 l.append(self.cl.get(self.nodeid, name))
1394 return l
1395 def items(self, protected=1):
1396 l = []
1397 for name in self.cl.getprops(protected=protected).keys():
1398 l.append((name, self.cl.get(self.nodeid, name)))
1399 return l
1400 def has_key(self, name):
1401 return self.cl.getprops().has_key(name)
1402 def get(self, name, default=None):
1403 if self.has_key(name):
1404 return self[name]
1405 else:
1406 return default
1407 def __getattr__(self, name):
1408 if self.__dict__.has_key(name):
1409 return self.__dict__[name]
1410 try:
1411 return self.cl.get(self.nodeid, name)
1412 except KeyError, value:
1413 # we trap this but re-raise it as AttributeError - all other
1414 # exceptions should pass through untrapped
1415 pass
1416 # nope, no such attribute
1417 raise AttributeError, str(value)
1418 def __getitem__(self, name):
1419 return self.cl.get(self.nodeid, name)
1420 def __setattr__(self, name, value):
1421 try:
1422 return self.cl.set(self.nodeid, **{name: value})
1423 except KeyError, value:
1424 raise AttributeError, str(value)
1425 def __setitem__(self, name, value):
1426 self.cl.set(self.nodeid, **{name: value})
1427 def history(self):
1428 return self.cl.history(self.nodeid)
1429 def retire(self):
1430 return self.cl.retire(self.nodeid)
1433 def Choice(name, db, *options):
1434 """Quick helper to create a simple class with choices
1435 """
1436 cl = Class(db, name, name=String(), order=String())
1437 for i in range(len(options)):
1438 cl.create(name=options[i], order=i)
1439 return Link(name)
1441 # vim: set filetype=python sts=4 sw=4 et si :