1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 """Hyperdatabase implementation, especially field types.
20 """
21 __docformat__ = 'restructuredtext'
23 # standard python modules
24 import os, re, shutil, weakref
25 # Python 2.3 ... 2.6 compatibility:
26 from roundup.anypy.sets_ import set
28 # roundup modules
29 import date, password
30 from support import ensureParentsExist, PrioList, sorted, reversed
31 from roundup.i18n import _
33 #
34 # Types
35 #
36 class _Type(object):
37 """A roundup property type."""
38 def __init__(self, required=False, default_value = None):
39 self.required = required
40 self.__default_value = default_value
41 def __repr__(self):
42 ' more useful for dumps '
43 return '<%s.%s>'%(self.__class__.__module__, self.__class__.__name__)
44 def get_default_value(self):
45 """The default value when creating a new instance of this property."""
46 return self.__default_value
47 def sort_repr (self, cls, val, name):
48 """Representation used for sorting. This should be a python
49 built-in type, otherwise sorting will take ages. Note that
50 individual backends may chose to use something different for
51 sorting as long as the outcome is the same.
52 """
53 return val
55 class String(_Type):
56 """An object designating a String property."""
57 def __init__(self, indexme='no', required=False, default_value = ""):
58 super(String, self).__init__(required, default_value)
59 self.indexme = indexme == 'yes'
60 def from_raw(self, value, propname='', **kw):
61 """fix the CRLF/CR -> LF stuff"""
62 if propname == 'content':
63 # Why oh why wasn't the FileClass content property a File
64 # type from the beginning?
65 return value
66 return fixNewlines(value)
67 def sort_repr (self, cls, val, name):
68 if not val:
69 return val
70 if name == 'id':
71 return int(val)
72 return val.lower()
74 class Password(_Type):
75 """An object designating a Password property."""
76 def from_raw(self, value, **kw):
77 if not value:
78 return None
79 try:
80 return password.Password(encrypted=value, strict=True)
81 except password.PasswordValueError, message:
82 raise HyperdbValueError, \
83 _('property %s: %s')%(kw['propname'], message)
85 def sort_repr (self, cls, val, name):
86 if not val:
87 return val
88 return str(val)
90 class Date(_Type):
91 """An object designating a Date property."""
92 def __init__(self, offset=None, required=False, default_value = None):
93 super(Date, self).__init__(required = required,
94 default_value = default_value)
95 self._offset = offset
96 def offset(self, db):
97 if self._offset is not None:
98 return self._offset
99 return db.getUserTimezone()
100 def from_raw(self, value, db, **kw):
101 try:
102 value = date.Date(value, self.offset(db))
103 except ValueError, message:
104 raise HyperdbValueError, _('property %s: %r is an invalid '\
105 'date (%s)')%(kw['propname'], value, message)
106 return value
107 def range_from_raw(self, value, db):
108 """return Range value from given raw value with offset correction"""
109 return date.Range(value, date.Date, offset=self.offset(db))
110 def sort_repr (self, cls, val, name):
111 if not val:
112 return val
113 return str(val)
115 class Interval(_Type):
116 """An object designating an Interval property."""
117 def from_raw(self, value, **kw):
118 try:
119 value = date.Interval(value)
120 except ValueError, message:
121 raise HyperdbValueError, _('property %s: %r is an invalid '\
122 'date interval (%s)')%(kw['propname'], value, message)
123 return value
124 def sort_repr (self, cls, val, name):
125 if not val:
126 return val
127 return val.as_seconds()
129 class _Pointer(_Type):
130 """An object designating a Pointer property that links or multilinks
131 to a node in a specified class."""
132 def __init__(self, classname, do_journal='yes', required=False,
133 default_value = None):
134 """ Default is to journal link and unlink events
135 """
136 super(_Pointer, self).__init__(required, default_value)
137 self.classname = classname
138 self.do_journal = do_journal == 'yes'
139 def __repr__(self):
140 """more useful for dumps. But beware: This is also used in schema
141 storage in SQL backends!
142 """
143 return '<%s.%s to "%s">'%(self.__class__.__module__,
144 self.__class__.__name__, self.classname)
146 class Link(_Pointer):
147 """An object designating a Link property that links to a
148 node in a specified class."""
149 def from_raw(self, value, db, propname, **kw):
150 if value == '-1' or not value:
151 value = None
152 else:
153 value = convertLinkValue(db, propname, self, value)
154 return value
155 def sort_repr (self, cls, val, name):
156 if not val:
157 return val
158 op = cls.labelprop()
159 if op == 'id':
160 return int(cls.get(val, op))
161 return cls.get(val, op)
163 class Multilink(_Pointer):
164 """An object designating a Multilink property that links
165 to nodes in a specified class.
167 "classname" indicates the class to link to
169 "do_journal" indicates whether the linked-to nodes should have
170 'link' and 'unlink' events placed in their journal
171 """
173 def __init__(self, classname, do_journal = 'yes', required = False):
175 super(Multilink, self).__init__(classname,
176 do_journal,
177 required = required,
178 default_value = [])
180 def from_raw(self, value, db, klass, propname, itemid, **kw):
181 if not value:
182 return []
184 # get the current item value if it's not a new item
185 if itemid and not itemid.startswith('-'):
186 curvalue = klass.get(itemid, propname)
187 else:
188 curvalue = []
190 # if the value is a comma-separated string then split it now
191 if isinstance(value, type('')):
192 value = value.split(',')
194 # handle each add/remove in turn
195 # keep an extra list for all items that are
196 # definitely in the new list (in case of e.g.
197 # <propname>=A,+B, which should replace the old
198 # list with A,B)
199 do_set = 1
200 newvalue = []
201 for item in value:
202 item = item.strip()
204 # skip blanks
205 if not item: continue
207 # handle +/-
208 remove = 0
209 if item.startswith('-'):
210 remove = 1
211 item = item[1:]
212 do_set = 0
213 elif item.startswith('+'):
214 item = item[1:]
215 do_set = 0
217 # look up the value
218 itemid = convertLinkValue(db, propname, self, item)
220 # perform the add/remove
221 if remove:
222 try:
223 curvalue.remove(itemid)
224 except ValueError:
225 raise HyperdbValueError, _('property %s: %r is not ' \
226 'currently an element')%(propname, item)
227 else:
228 newvalue.append(itemid)
229 if itemid not in curvalue:
230 curvalue.append(itemid)
232 # that's it, set the new Multilink property value,
233 # or overwrite it completely
234 if do_set:
235 value = newvalue
236 else:
237 value = curvalue
239 # TODO: one day, we'll switch to numeric ids and this will be
240 # unnecessary :(
241 value = [int(x) for x in value]
242 value.sort()
243 value = [str(x) for x in value]
244 return value
246 def sort_repr (self, cls, val, name):
247 if not val:
248 return val
249 op = cls.labelprop()
250 if op == 'id':
251 return [int(cls.get(v, op)) for v in val]
252 return [cls.get(v, op) for v in val]
254 class Boolean(_Type):
255 """An object designating a boolean property"""
256 def from_raw(self, value, **kw):
257 value = value.strip()
258 # checked is a common HTML checkbox value
259 value = value.lower() in ('checked', 'yes', 'true', 'on', '1')
260 return value
262 class Number(_Type):
263 """An object designating a numeric property"""
264 def from_raw(self, value, **kw):
265 value = value.strip()
266 try:
267 value = float(value)
268 except ValueError:
269 raise HyperdbValueError, _('property %s: %r is not a number')%(
270 kw['propname'], value)
271 return value
272 #
273 # Support for splitting designators
274 #
275 class DesignatorError(ValueError):
276 pass
277 def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')):
278 """ Take a foo123 and return ('foo', 123)
279 """
280 m = dre.match(designator)
281 if m is None:
282 raise DesignatorError, _('"%s" not a node designator')%designator
283 return m.group(1), m.group(2)
285 class Proptree(object):
286 """ Simple tree data structure for optimizing searching of
287 properties. Each node in the tree represents a roundup Class
288 Property that has to be navigated for finding the given search
289 or sort properties. The need_for attribute is used for
290 distinguishing nodes in the tree used for sorting, searching or
291 retrieval: The attribute is a dictionary containing one or several
292 of the values 'sort', 'search', 'retrieve'.
294 The Proptree is also used for transitively searching attributes for
295 backends that do not support transitive search (e.g. anydbm). The
296 _val attribute with set_val is used for this.
297 """
299 def __init__(self, db, cls, name, props, parent=None, retr=False):
300 self.db = db
301 self.name = name
302 self.props = props
303 self.parent = parent
304 self._val = None
305 self.has_values = False
306 self.cls = cls
307 self.classname = None
308 self.uniqname = None
309 self.children = []
310 self.sortattr = []
311 self.propdict = {}
312 self.need_for = {'search' : True}
313 self.sort_direction = None
314 self.sort_ids = None
315 self.sort_ids_needed = False
316 self.sort_result = None
317 self.attr_sort_done = False
318 self.tree_sort_done = False
319 self.propclass = None
320 self.orderby = []
321 self.sql_idx = None # index of retrieved column in sql result
322 if parent:
323 self.root = parent.root
324 self.depth = parent.depth + 1
325 else:
326 self.root = self
327 self.seqno = 1
328 self.depth = 0
329 self.need_for['sort'] = True
330 self.id = self.root.seqno
331 self.root.seqno += 1
332 if self.cls:
333 self.classname = self.cls.classname
334 self.uniqname = '%s%s' % (self.cls.classname, self.id)
335 if not self.parent:
336 self.uniqname = self.cls.classname
337 if retr:
338 self.append_retr_props()
340 def append(self, name, need_for='search', retr=False):
341 """Append a property to self.children. Will create a new
342 propclass for the child.
343 """
344 if name in self.propdict:
345 pt = self.propdict[name]
346 pt.need_for[need_for] = True
347 if retr and isinstance(pt.propclass, Link):
348 pt.append_retr_props()
349 return pt
350 propclass = self.props[name]
351 cls = None
352 props = None
353 if isinstance(propclass, (Link, Multilink)):
354 cls = self.db.getclass(propclass.classname)
355 props = cls.getprops()
356 child = self.__class__(self.db, cls, name, props, parent = self)
357 child.need_for = {need_for : True}
358 child.propclass = propclass
359 self.children.append(child)
360 self.propdict[name] = child
361 if retr and isinstance(child.propclass, Link):
362 child.append_retr_props()
363 return child
365 def append_retr_props(self):
366 """Append properties for retrieval."""
367 for name, prop in self.cls.getprops(protected=1).iteritems():
368 if isinstance(prop, Multilink):
369 continue
370 self.append(name, need_for='retrieve')
372 def compute_sort_done(self, mlseen=False):
373 """ Recursively check if attribute is needed for sorting
374 ('sort' in self.need_for) or all children have tree_sort_done set and
375 sort_ids_needed unset: set self.tree_sort_done if one of the conditions
376 holds. Also remove sort_ids_needed recursively once having seen a
377 Multilink.
378 """
379 if isinstance (self.propclass, Multilink):
380 mlseen = True
381 if mlseen:
382 self.sort_ids_needed = False
383 self.tree_sort_done = True
384 for p in self.children:
385 p.compute_sort_done(mlseen)
386 if not p.tree_sort_done:
387 self.tree_sort_done = False
388 if 'sort' not in self.need_for:
389 self.tree_sort_done = True
390 if mlseen:
391 self.tree_sort_done = False
393 def ancestors(self):
394 p = self
395 while p.parent:
396 yield p
397 p = p.parent
399 def search(self, search_matches=None, sort=True):
400 """ Recursively search for the given properties in a proptree.
401 Once all properties are non-transitive, the search generates a
402 simple _filter call which does the real work
403 """
404 filterspec = {}
405 for p in self.children:
406 if 'search' in p.need_for:
407 if p.children:
408 p.search(sort = False)
409 filterspec[p.name] = p.val
410 self.val = self.cls._filter(search_matches, filterspec, sort and self)
411 return self.val
413 def sort (self, ids=None):
414 """ Sort ids by the order information stored in self. With
415 optimisations: Some order attributes may be precomputed (by the
416 backend) and some properties may already be sorted.
417 """
418 if ids is None:
419 ids = self.val
420 if self.sortattr and [s for s in self.sortattr if not s.attr_sort_done]:
421 return self._searchsort(ids, True, True)
422 return ids
424 def sortable_children(self, intermediate=False):
425 """ All children needed for sorting. If intermediate is True,
426 intermediate nodes (not being a sort attribute) are returned,
427 too.
428 """
429 return [p for p in self.children
430 if 'sort' in p.need_for and (intermediate or p.sort_direction)]
432 def __iter__(self):
433 """ Yield nodes in depth-first order -- visited nodes first """
434 for p in self.children:
435 yield p
436 for c in p:
437 yield c
439 def _get (self, ids):
440 """Lookup given ids -- possibly a list of list. We recurse until
441 we have a list of ids.
442 """
443 if not ids:
444 return ids
445 if isinstance (ids[0], list):
446 cids = [self._get(i) for i in ids]
447 else:
448 cids = [i and self.parent.cls.get(i, self.name) for i in ids]
449 if self.sortattr:
450 cids = [self._searchsort(i, False, True) for i in cids]
451 return cids
453 def _searchsort(self, ids=None, update=True, dosort=True):
454 """ Recursively compute the sort attributes. Note that ids
455 may be a deeply nested list of lists of ids if several
456 multilinks are encountered on the way from the root to an
457 individual attribute. We make sure that everything is properly
458 sorted on the way up. Note that the individual backend may
459 already have precomputed self.result or self.sort_ids. In this
460 case we do nothing for existing sa.result and recurse further if
461 self.sort_ids is available.
463 Yech, Multilinks: This gets especially complicated if somebody
464 sorts by different attributes of the same multilink (or
465 transitively across several multilinks). My use-case is sorting
466 by issue.messages.author and (reverse) by issue.messages.date.
467 In this case we sort the messages by author and date and use
468 this sorted list twice for sorting issues. This means that
469 issues are sorted by author and then by the time of the messages
470 *of this author*. Probably what the user intends in that case,
471 so we do *not* use two sorted lists of messages, one sorted by
472 author and one sorted by date for sorting issues.
473 """
474 for pt in self.sortable_children(intermediate = True):
475 # ids can be an empty list
476 if pt.tree_sort_done or not ids:
477 continue
478 if pt.sort_ids: # cached or computed by backend
479 cids = pt.sort_ids
480 else:
481 cids = pt._get(ids)
482 if pt.sort_direction and not pt.sort_result:
483 sortrep = pt.propclass.sort_repr
484 pt.sort_result = pt._sort_repr(sortrep, cids)
485 pt.sort_ids = cids
486 if pt.children:
487 pt._searchsort(cids, update, False)
488 if self.sortattr and dosort:
489 ids = self._sort(ids)
490 if not update:
491 for pt in self.sortable_children(intermediate = True):
492 pt.sort_ids = None
493 for pt in self.sortattr:
494 pt.sort_result = None
495 return ids
497 def _set_val(self, val):
498 """Check if self._val is already defined. If yes, we compute the
499 intersection of the old and the new value(s)
500 """
501 if self.has_values:
502 v = self._val
503 if not isinstance(self._val, type([])):
504 v = [self._val]
505 vals = set(v)
506 vals.intersection_update(val)
507 self._val = [v for v in vals]
508 else:
509 self._val = val
510 self.has_values = True
512 val = property(lambda self: self._val, _set_val)
514 def _sort(self, val):
515 """Finally sort by the given sortattr.sort_result. Note that we
516 do not sort by attrs having attr_sort_done set. The caller is
517 responsible for setting attr_sort_done only for trailing
518 attributes (otherwise the sort order is wrong). Since pythons
519 sort is stable, we can sort already sorted lists without
520 destroying the sort-order for items that compare equal with the
521 current sort.
523 Sorting-Strategy: We sort repeatedly by different sort-keys from
524 right to left. Since pythons sort is stable, we can safely do
525 that. An optimisation is a "run-length encoding" of the
526 sort-directions: If several sort attributes sort in the same
527 direction we can combine them into a single sort. Note that
528 repeated sorting is probably more efficient than using
529 compare-methods in python due to the overhead added by compare
530 methods.
531 """
532 if not val:
533 return val
534 sortattr = []
535 directions = []
536 dir_idx = []
537 idx = 0
538 curdir = None
539 for sa in self.sortattr:
540 if sa.attr_sort_done:
541 break
542 if sortattr:
543 assert len(sortattr[0]) == len(sa.sort_result)
544 sortattr.append (sa.sort_result)
545 if curdir != sa.sort_direction:
546 dir_idx.append (idx)
547 directions.append (sa.sort_direction)
548 curdir = sa.sort_direction
549 idx += 1
550 sortattr.append (val)
551 sortattr = zip (*sortattr)
552 for dir, i in reversed(zip(directions, dir_idx)):
553 rev = dir == '-'
554 sortattr = sorted (sortattr, key = lambda x:x[i:idx], reverse = rev)
555 idx = i
556 return [x[-1] for x in sortattr]
558 def _sort_repr(self, sortrep, ids):
559 """Call sortrep for given ids -- possibly a list of list. We
560 recurse until we have a list of ids.
561 """
562 if not ids:
563 return ids
564 if isinstance (ids[0], list):
565 res = [self._sort_repr(sortrep, i) for i in ids]
566 else:
567 res = [sortrep(self.cls, i, self.name) for i in ids]
568 return res
570 def __repr__(self):
571 r = ["proptree:" + self.name]
572 for n in self:
573 r.append("proptree:" + " " * n.depth + n.name)
574 return '\n'.join(r)
575 __str__ = __repr__
577 #
578 # the base Database class
579 #
580 class DatabaseError(ValueError):
581 """Error to be raised when there is some problem in the database code
582 """
583 pass
584 class Database:
585 """A database for storing records containing flexible data types.
587 This class defines a hyperdatabase storage layer, which the Classes use to
588 store their data.
591 Transactions
592 ------------
593 The Database should support transactions through the commit() and
594 rollback() methods. All other Database methods should be transaction-aware,
595 using data from the current transaction before looking up the database.
597 An implementation must provide an override for the get() method so that the
598 in-database value is returned in preference to the in-transaction value.
599 This is necessary to determine if any values have changed during a
600 transaction.
603 Implementation
604 --------------
606 All methods except __repr__ must be implemented by a concrete backend Database.
608 """
610 # flag to set on retired entries
611 RETIRED_FLAG = '__hyperdb_retired'
613 BACKEND_MISSING_STRING = None
614 BACKEND_MISSING_NUMBER = None
615 BACKEND_MISSING_BOOLEAN = None
617 def __init__(self, config, journaltag=None):
618 """Open a hyperdatabase given a specifier to some storage.
620 The 'storagelocator' is obtained from config.DATABASE.
621 The meaning of 'storagelocator' depends on the particular
622 implementation of the hyperdatabase. It could be a file name,
623 a directory path, a socket descriptor for a connection to a
624 database over the network, etc.
626 The 'journaltag' is a token that will be attached to the journal
627 entries for any edits done on the database. If 'journaltag' is
628 None, the database is opened in read-only mode: the Class.create(),
629 Class.set(), and Class.retire() methods are disabled.
630 """
631 raise NotImplementedError
633 def post_init(self):
634 """Called once the schema initialisation has finished.
635 If 'refresh' is true, we want to rebuild the backend
636 structures.
637 """
638 raise NotImplementedError
640 def refresh_database(self):
641 """Called to indicate that the backend should rebuild all tables
642 and structures. Not called in normal usage."""
643 raise NotImplementedError
645 def __getattr__(self, classname):
646 """A convenient way of calling self.getclass(classname)."""
647 raise NotImplementedError
649 def addclass(self, cl):
650 """Add a Class to the hyperdatabase.
651 """
652 raise NotImplementedError
654 def getclasses(self):
655 """Return a list of the names of all existing classes."""
656 raise NotImplementedError
658 def getclass(self, classname):
659 """Get the Class object representing a particular class.
661 If 'classname' is not a valid class name, a KeyError is raised.
662 """
663 raise NotImplementedError
665 def clear(self):
666 """Delete all database contents.
667 """
668 raise NotImplementedError
670 def getclassdb(self, classname, mode='r'):
671 """Obtain a connection to the class db that will be used for
672 multiple actions.
673 """
674 raise NotImplementedError
676 def addnode(self, classname, nodeid, node):
677 """Add the specified node to its class's db.
678 """
679 raise NotImplementedError
681 def serialise(self, classname, node):
682 """Copy the node contents, converting non-marshallable data into
683 marshallable data.
684 """
685 return node
687 def setnode(self, classname, nodeid, node):
688 """Change the specified node.
689 """
690 raise NotImplementedError
692 def unserialise(self, classname, node):
693 """Decode the marshalled node data
694 """
695 return node
697 def getnode(self, classname, nodeid):
698 """Get a node from the database.
700 'cache' exists for backwards compatibility, and is not used.
701 """
702 raise NotImplementedError
704 def hasnode(self, classname, nodeid):
705 """Determine if the database has a given node.
706 """
707 raise NotImplementedError
709 def countnodes(self, classname):
710 """Count the number of nodes that exist for a particular Class.
711 """
712 raise NotImplementedError
714 def storefile(self, classname, nodeid, property, content):
715 """Store the content of the file in the database.
717 The property may be None, in which case the filename does not
718 indicate which property is being saved.
719 """
720 raise NotImplementedError
722 def getfile(self, classname, nodeid, property):
723 """Get the content of the file in the database.
724 """
725 raise NotImplementedError
727 def addjournal(self, classname, nodeid, action, params):
728 """ Journal the Action
729 'action' may be:
731 'create' or 'set' -- 'params' is a dictionary of property values
732 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
733 'retire' -- 'params' is None
734 """
735 raise NotImplementedError
737 def getjournal(self, classname, nodeid):
738 """ get the journal for id
739 """
740 raise NotImplementedError
742 def pack(self, pack_before):
743 """ pack the database
744 """
745 raise NotImplementedError
747 def commit(self):
748 """ Commit the current transactions.
750 Save all data changed since the database was opened or since the
751 last commit() or rollback().
753 fail_ok indicates that the commit is allowed to fail. This is used
754 in the web interface when committing cleaning of the session
755 database. We don't care if there's a concurrency issue there.
757 The only backend this seems to affect is postgres.
758 """
759 raise NotImplementedError
761 def rollback(self):
762 """ Reverse all actions from the current transaction.
764 Undo all the changes made since the database was opened or the last
765 commit() or rollback() was performed.
766 """
767 raise NotImplementedError
769 def close(self):
770 """Close the database.
772 This method must be called at the end of processing.
774 """
776 def iter_roles(roles):
777 ''' handle the text processing of turning the roles list
778 into something python can use more easily
779 '''
780 if not roles or not roles.strip():
781 raise StopIteration, "Empty roles given"
782 for role in [x.lower().strip() for x in roles.split(',')]:
783 yield role
786 #
787 # The base Class class
788 #
789 class Class:
790 """ The handle to a particular class of nodes in a hyperdatabase.
792 All methods except __repr__ and getnode must be implemented by a
793 concrete backend Class.
794 """
796 def __init__(self, db, classname, **properties):
797 """Create a new class with a given name and property specification.
799 'classname' must not collide with the name of an existing class,
800 or a ValueError is raised. The keyword arguments in 'properties'
801 must map names to property objects, or a TypeError is raised.
802 """
803 for name in 'creation activity creator actor'.split():
804 if properties.has_key(name):
805 raise ValueError, '"creation", "activity", "creator" and '\
806 '"actor" are reserved'
808 self.classname = classname
809 self.properties = properties
810 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
811 self.key = ''
813 # should we journal changes (default yes)
814 self.do_journal = 1
816 # do the db-related init stuff
817 db.addclass(self)
819 actions = "create set retire restore".split()
820 self.auditors = dict([(a, PrioList()) for a in actions])
821 self.reactors = dict([(a, PrioList()) for a in actions])
823 def __repr__(self):
824 """Slightly more useful representation
825 """
826 return '<hyperdb.Class "%s">'%self.classname
828 # Editing nodes:
830 def create(self, **propvalues):
831 """Create a new node of this class and return its id.
833 The keyword arguments in 'propvalues' map property names to values.
835 The values of arguments must be acceptable for the types of their
836 corresponding properties or a TypeError is raised.
838 If this class has a key property, it must be present and its value
839 must not collide with other key strings or a ValueError is raised.
841 Any other properties on this class that are missing from the
842 'propvalues' dictionary are set to None.
844 If an id in a link or multilink property does not refer to a valid
845 node, an IndexError is raised.
846 """
847 raise NotImplementedError
849 _marker = []
850 def get(self, nodeid, propname, default=_marker, cache=1):
851 """Get the value of a property on an existing node of this class.
853 'nodeid' must be the id of an existing node of this class or an
854 IndexError is raised. 'propname' must be the name of a property
855 of this class or a KeyError is raised.
857 'cache' exists for backwards compatibility, and is not used.
858 """
859 raise NotImplementedError
861 # not in spec
862 def getnode(self, nodeid):
863 """ Return a convenience wrapper for the node.
865 'nodeid' must be the id of an existing node of this class or an
866 IndexError is raised.
868 'cache' exists for backwards compatibility, and is not used.
869 """
870 return Node(self, nodeid)
872 def getnodeids(self, retired=None):
873 """Retrieve all the ids of the nodes for a particular Class.
874 """
875 raise NotImplementedError
877 def set(self, nodeid, **propvalues):
878 """Modify a property on an existing node of this class.
880 'nodeid' must be the id of an existing node of this class or an
881 IndexError is raised.
883 Each key in 'propvalues' must be the name of a property of this
884 class or a KeyError is raised.
886 All values in 'propvalues' must be acceptable types for their
887 corresponding properties or a TypeError is raised.
889 If the value of the key property is set, it must not collide with
890 other key strings or a ValueError is raised.
892 If the value of a Link or Multilink property contains an invalid
893 node id, a ValueError is raised.
894 """
895 raise NotImplementedError
897 def retire(self, nodeid):
898 """Retire a node.
900 The properties on the node remain available from the get() method,
901 and the node's id is never reused.
903 Retired nodes are not returned by the find(), list(), or lookup()
904 methods, and other nodes may reuse the values of their key properties.
905 """
906 raise NotImplementedError
908 def restore(self, nodeid):
909 """Restpre a retired node.
911 Make node available for all operations like it was before retirement.
912 """
913 raise NotImplementedError
915 def is_retired(self, nodeid):
916 """Return true if the node is rerired
917 """
918 raise NotImplementedError
920 def destroy(self, nodeid):
921 """Destroy a node.
923 WARNING: this method should never be used except in extremely rare
924 situations where there could never be links to the node being
925 deleted
927 WARNING: use retire() instead
929 WARNING: the properties of this node will not be available ever again
931 WARNING: really, use retire() instead
933 Well, I think that's enough warnings. This method exists mostly to
934 support the session storage of the cgi interface.
936 The node is completely removed from the hyperdb, including all journal
937 entries. It will no longer be available, and will generally break code
938 if there are any references to the node.
939 """
941 def history(self, nodeid):
942 """Retrieve the journal of edits on a particular node.
944 'nodeid' must be the id of an existing node of this class or an
945 IndexError is raised.
947 The returned list contains tuples of the form
949 (date, tag, action, params)
951 'date' is a Timestamp object specifying the time of the change and
952 'tag' is the journaltag specified when the database was opened.
953 """
954 if not self.do_journal:
955 raise ValueError('Journalling is disabled for this class')
956 return self.db.getjournal(self.classname, nodeid)
958 # Locating nodes:
959 def hasnode(self, nodeid):
960 """Determine if the given nodeid actually exists
961 """
962 raise NotImplementedError
964 def setkey(self, propname):
965 """Select a String property of this class to be the key property.
967 'propname' must be the name of a String property of this class or
968 None, or a TypeError is raised. The values of the key property on
969 all existing nodes must be unique or a ValueError is raised.
970 """
971 raise NotImplementedError
973 def setlabelprop(self, labelprop):
974 """Set the label property. Used for override of labelprop
975 resolution order.
976 """
977 if labelprop not in self.getprops():
978 raise ValueError, _("Not a property name: %s") % labelprop
979 self._labelprop = labelprop
981 def setorderprop(self, orderprop):
982 """Set the order property. Used for override of orderprop
983 resolution order
984 """
985 if orderprop not in self.getprops():
986 raise ValueError, _("Not a property name: %s") % orderprop
987 self._orderprop = orderprop
989 def getkey(self):
990 """Return the name of the key property for this class or None."""
991 raise NotImplementedError
993 def labelprop(self, default_to_id=0):
994 """Return the property name for a label for the given node.
996 This method attempts to generate a consistent label for the node.
997 It tries the following in order:
999 0. self._labelprop if set
1000 1. key property
1001 2. "name" property
1002 3. "title" property
1003 4. first property from the sorted property name list
1004 """
1005 if hasattr(self, '_labelprop'):
1006 return self._labelprop
1007 k = self.getkey()
1008 if k:
1009 return k
1010 props = self.getprops()
1011 if props.has_key('name'):
1012 return 'name'
1013 elif props.has_key('title'):
1014 return 'title'
1015 if default_to_id:
1016 return 'id'
1017 props = props.keys()
1018 props.sort()
1019 return props[0]
1021 def orderprop(self):
1022 """Return the property name to use for sorting for the given node.
1024 This method computes the property for sorting.
1025 It tries the following in order:
1027 0. self._orderprop if set
1028 1. "order" property
1029 2. self.labelprop()
1030 """
1032 if hasattr(self, '_orderprop'):
1033 return self._orderprop
1034 props = self.getprops()
1035 if props.has_key('order'):
1036 return 'order'
1037 return self.labelprop()
1039 def lookup(self, keyvalue):
1040 """Locate a particular node by its key property and return its id.
1042 If this class has no key property, a TypeError is raised. If the
1043 'keyvalue' matches one of the values for the key property among
1044 the nodes in this class, the matching node's id is returned;
1045 otherwise a KeyError is raised.
1046 """
1047 raise NotImplementedError
1049 def find(self, **propspec):
1050 """Get the ids of nodes in this class which link to the given nodes.
1052 'propspec' consists of keyword args propname={nodeid:1,}
1053 'propname' must be the name of a property in this class, or a
1054 KeyError is raised. That property must be a Link or Multilink
1055 property, or a TypeError is raised.
1057 Any node in this class whose 'propname' property links to any of the
1058 nodeids will be returned. Used by the full text indexing, which knows
1059 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1060 issues:
1062 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1063 """
1064 raise NotImplementedError
1066 def _filter(self, search_matches, filterspec, sort=(None,None),
1067 group=(None,None)):
1068 """For some backends this implements the non-transitive
1069 search, for more information see the filter method.
1070 """
1071 raise NotImplementedError
1073 def _proptree(self, filterspec, sortattr=[], retr=False):
1074 """Build a tree of all transitive properties in the given
1075 filterspec.
1076 If we retrieve (retr is True) linked items we don't follow
1077 across multilinks. We also don't follow if the searched value
1078 can contain NULL values.
1079 """
1080 proptree = Proptree(self.db, self, '', self.getprops(), retr=retr)
1081 for key, v in filterspec.iteritems():
1082 keys = key.split('.')
1083 p = proptree
1084 mlseen = False
1085 for k in keys:
1086 if isinstance (p.propclass, Multilink):
1087 mlseen = True
1088 isnull = v == '-1' or v is None
1089 nullin = isinstance(v, type([])) and ('-1' in v or None in v)
1090 r = retr and not mlseen and not isnull and not nullin
1091 p = p.append(k, retr=r)
1092 p.val = v
1093 multilinks = {}
1094 for s in sortattr:
1095 keys = s[1].split('.')
1096 p = proptree
1097 mlseen = False
1098 for k in keys:
1099 if isinstance (p.propclass, Multilink):
1100 mlseen = True
1101 r = retr and not mlseen
1102 p = p.append(k, need_for='sort', retr=r)
1103 if isinstance (p.propclass, Multilink):
1104 multilinks[p] = True
1105 if p.cls:
1106 p = p.append(p.cls.orderprop(), need_for='sort')
1107 if p.sort_direction: # if an orderprop is also specified explicitly
1108 continue
1109 p.sort_direction = s[0]
1110 proptree.sortattr.append (p)
1111 for p in multilinks.iterkeys():
1112 sattr = {}
1113 for c in p:
1114 if c.sort_direction:
1115 sattr [c] = True
1116 for sa in proptree.sortattr:
1117 if sa in sattr:
1118 p.sortattr.append (sa)
1119 return proptree
1121 def get_transitive_prop(self, propname_path, default = None):
1122 """Expand a transitive property (individual property names
1123 separated by '.' into a new property at the end of the path. If
1124 one of the names does not refer to a valid property, we return
1125 None.
1126 Example propname_path (for class issue): "messages.author"
1127 """
1128 props = self.db.getclass(self.classname).getprops()
1129 for k in propname_path.split('.'):
1130 try:
1131 prop = props[k]
1132 except (KeyError, TypeError):
1133 return default
1134 cl = getattr(prop, 'classname', None)
1135 props = None
1136 if cl:
1137 props = self.db.getclass(cl).getprops()
1138 return prop
1140 def _sortattr(self, sort=[], group=[]):
1141 """Build a single list of sort attributes in the correct order
1142 with sanity checks (no duplicate properties) included. Always
1143 sort last by id -- if id is not already in sortattr.
1144 """
1145 seen = {}
1146 sortattr = []
1147 for srt in group, sort:
1148 if not isinstance(srt, list):
1149 srt = [srt]
1150 for s in srt:
1151 if s[1] and s[1] not in seen:
1152 sortattr.append((s[0] or '+', s[1]))
1153 seen[s[1]] = True
1154 if 'id' not in seen :
1155 sortattr.append(('+', 'id'))
1156 return sortattr
1158 def filter(self, search_matches, filterspec, sort=[], group=[]):
1159 """Return a list of the ids of the active nodes in this class that
1160 match the 'filter' spec, sorted by the group spec and then the
1161 sort spec.
1163 "filterspec" is {propname: value(s)}
1165 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
1166 or None and prop is a prop name or None. Note that for
1167 backward-compatibility reasons a single (dir, prop) tuple is
1168 also allowed.
1170 "search_matches" is a container type
1172 The filter must match all properties specificed. If the property
1173 value to match is a list:
1175 1. String properties must match all elements in the list, and
1176 2. Other properties must match any of the elements in the list.
1178 Note that now the propname in filterspec and prop in a
1179 sort/group spec may be transitive, i.e., it may contain
1180 properties of the form link.link.link.name, e.g. you can search
1181 for all issues where a message was added by a certain user in
1182 the last week with a filterspec of
1183 {'messages.author' : '42', 'messages.creation' : '.-1w;'}
1185 Implementation note:
1186 This implements a non-optimized version of Transitive search
1187 using _filter implemented in a backend class. A more efficient
1188 version can be implemented in the individual backends -- e.g.,
1189 an SQL backend will want to create a single SQL statement and
1190 override the filter method instead of implementing _filter.
1191 """
1192 sortattr = self._sortattr(sort = sort, group = group)
1193 proptree = self._proptree(filterspec, sortattr)
1194 proptree.search(search_matches)
1195 return proptree.sort()
1197 # non-optimized filter_iter, a backend may chose to implement a
1198 # better version that provides a real iterator that pre-fills the
1199 # cache for each id returned. Note that the filter_iter doesn't
1200 # promise to correctly sort by multilink (which isn't sane to do
1201 # anyway).
1202 filter_iter = filter
1204 def count(self):
1205 """Get the number of nodes in this class.
1207 If the returned integer is 'numnodes', the ids of all the nodes
1208 in this class run from 1 to numnodes, and numnodes+1 will be the
1209 id of the next node to be created in this class.
1210 """
1211 raise NotImplementedError
1213 # Manipulating properties:
1214 def getprops(self, protected=1):
1215 """Return a dictionary mapping property names to property objects.
1216 If the "protected" flag is true, we include protected properties -
1217 those which may not be modified.
1218 """
1219 raise NotImplementedError
1221 def get_required_props(self, propnames = []):
1222 """Return a dict of property names mapping to property objects.
1223 All properties that have the "required" flag set will be
1224 returned in addition to all properties in the propnames
1225 parameter.
1226 """
1227 props = self.getprops(protected = False)
1228 pdict = dict([(p, props[p]) for p in propnames])
1229 pdict.update([(k, v) for k, v in props.iteritems() if v.required])
1230 return pdict
1232 def addprop(self, **properties):
1233 """Add properties to this class.
1235 The keyword arguments in 'properties' must map names to property
1236 objects, or a TypeError is raised. None of the keys in 'properties'
1237 may collide with the names of existing properties, or a ValueError
1238 is raised before any properties have been added.
1239 """
1240 raise NotImplementedError
1242 def index(self, nodeid):
1243 """Add (or refresh) the node to search indexes"""
1244 raise NotImplementedError
1246 #
1247 # Detector interface
1248 #
1249 def audit(self, event, detector, priority = 100):
1250 """Register an auditor detector"""
1251 self.auditors[event].append((priority, detector.__name__, detector))
1253 def fireAuditors(self, event, nodeid, newvalues):
1254 """Fire all registered auditors"""
1255 for prio, name, audit in self.auditors[event]:
1256 audit(self.db, self, nodeid, newvalues)
1258 def react(self, event, detector, priority = 100):
1259 """Register a reactor detector"""
1260 self.reactors[event].append((priority, detector.__name__, detector))
1262 def fireReactors(self, event, nodeid, oldvalues):
1263 """Fire all registered reactors"""
1264 for prio, name, react in self.reactors[event]:
1265 react(self.db, self, nodeid, oldvalues)
1267 #
1268 # import / export support
1269 #
1270 def export_propnames(self):
1271 """List the property names for export from this Class"""
1272 propnames = self.getprops().keys()
1273 propnames.sort()
1274 return propnames
1276 def import_journals(self, entries):
1277 """Import a class's journal.
1279 Uses setjournal() to set the journal for each item.
1280 Strategy for import: Sort first by id, then import journals for
1281 each id, this way the memory footprint is a lot smaller than the
1282 initial implementation which stored everything in a big hash by
1283 id and then proceeded to import journals for each id."""
1284 properties = self.getprops()
1285 a = []
1286 for l in entries:
1287 # first element in sorted list is the (numeric) id
1288 # in python2.4 and up we would use sorted with a key...
1289 a.append ((int (l [0].strip ("'")), l))
1290 a.sort ()
1293 last = 0
1294 r = []
1295 for n, l in a:
1296 nodeid, jdate, user, action, params = map(eval, l)
1297 assert (str(n) == nodeid)
1298 if n != last:
1299 if r:
1300 self.db.setjournal(self.classname, str(last), r)
1301 last = n
1302 r = []
1304 if action == 'set':
1305 for propname, value in params.iteritems():
1306 prop = properties[propname]
1307 if value is None:
1308 pass
1309 elif isinstance(prop, Date):
1310 value = date.Date(value)
1311 elif isinstance(prop, Interval):
1312 value = date.Interval(value)
1313 elif isinstance(prop, Password):
1314 value = password.JournalPassword(encrypted=value)
1315 params[propname] = value
1316 elif action == 'create' and params:
1317 # old tracker with data stored in the create!
1318 params = {}
1319 r.append((nodeid, date.Date(jdate), user, action, params))
1320 if r:
1321 self.db.setjournal(self.classname, nodeid, r)
1323 #
1324 # convenience methods
1325 #
1326 def get_roles(self, nodeid):
1327 """Return iterator for all roles for this nodeid.
1329 Yields string-processed roles.
1330 This method can be overridden to provide a hook where we can
1331 insert other permission models (e.g. get roles from database)
1332 In standard schemas only a user has a roles property but
1333 this may be different in customized schemas.
1334 Note that this is the *central place* where role
1335 processing happens!
1336 """
1337 node = self.db.getnode(self.classname, nodeid)
1338 return iter_roles(node['roles'])
1340 def has_role(self, nodeid, *roles):
1341 '''See if this node has any roles that appear in roles.
1343 For convenience reasons we take a list.
1344 In standard schemas only a user has a roles property but
1345 this may be different in customized schemas.
1346 '''
1347 roles = dict.fromkeys ([r.strip().lower() for r in roles])
1348 for role in self.get_roles(nodeid):
1349 if role in roles:
1350 return True
1351 return False
1354 class HyperdbValueError(ValueError):
1355 """ Error converting a raw value into a Hyperdb value """
1356 pass
1358 def convertLinkValue(db, propname, prop, value, idre=re.compile('^\d+$')):
1359 """ Convert the link value (may be id or key value) to an id value. """
1360 linkcl = db.classes[prop.classname]
1361 if not idre.match(value):
1362 if linkcl.getkey():
1363 try:
1364 value = linkcl.lookup(value)
1365 except KeyError, message:
1366 raise HyperdbValueError, _('property %s: %r is not a %s.')%(
1367 propname, value, prop.classname)
1368 else:
1369 raise HyperdbValueError, _('you may only enter ID values '\
1370 'for property %s')%propname
1371 return value
1373 def fixNewlines(text):
1374 """ Homogenise line endings.
1376 Different web clients send different line ending values, but
1377 other systems (eg. email) don't necessarily handle those line
1378 endings. Our solution is to convert all line endings to LF.
1379 """
1380 text = text.replace('\r\n', '\n')
1381 return text.replace('\r', '\n')
1383 def rawToHyperdb(db, klass, itemid, propname, value, **kw):
1384 """ Convert the raw (user-input) value to a hyperdb-storable value. The
1385 value is for the "propname" property on itemid (may be None for a
1386 new item) of "klass" in "db".
1388 The value is usually a string, but in the case of multilink inputs
1389 it may be either a list of strings or a string with comma-separated
1390 values.
1391 """
1392 properties = klass.getprops()
1394 # ensure it's a valid property name
1395 propname = propname.strip()
1396 try:
1397 proptype = properties[propname]
1398 except KeyError:
1399 raise HyperdbValueError, _('%r is not a property of %s')%(propname,
1400 klass.classname)
1402 # if we got a string, strip it now
1403 if isinstance(value, type('')):
1404 value = value.strip()
1406 # convert the input value to a real property value
1407 value = proptype.from_raw(value, db=db, klass=klass,
1408 propname=propname, itemid=itemid, **kw)
1410 return value
1412 class FileClass:
1413 """ A class that requires the "content" property and stores it on
1414 disk.
1415 """
1416 default_mime_type = 'text/plain'
1418 def __init__(self, db, classname, **properties):
1419 """The newly-created class automatically includes the "content"
1420 property.
1421 """
1422 if not properties.has_key('content'):
1423 properties['content'] = String(indexme='yes')
1425 def export_propnames(self):
1426 """ Don't export the "content" property
1427 """
1428 propnames = self.getprops().keys()
1429 propnames.remove('content')
1430 propnames.sort()
1431 return propnames
1433 def exportFilename(self, dirname, nodeid):
1434 subdir_filename = self.db.subdirFilename(self.classname, nodeid)
1435 return os.path.join(dirname, self.classname+'-files', subdir_filename)
1437 def export_files(self, dirname, nodeid):
1438 """ Export the "content" property as a file, not csv column
1439 """
1440 source = self.db.filename(self.classname, nodeid)
1442 dest = self.exportFilename(dirname, nodeid)
1443 ensureParentsExist(dest)
1444 shutil.copyfile(source, dest)
1446 def import_files(self, dirname, nodeid):
1447 """ Import the "content" property as a file
1448 """
1449 source = self.exportFilename(dirname, nodeid)
1451 dest = self.db.filename(self.classname, nodeid, create=1)
1452 ensureParentsExist(dest)
1453 shutil.copyfile(source, dest)
1455 mime_type = None
1456 props = self.getprops()
1457 if props.has_key('type'):
1458 mime_type = self.get(nodeid, 'type')
1459 if not mime_type:
1460 mime_type = self.default_mime_type
1461 if props['content'].indexme:
1462 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1463 self.get(nodeid, 'content'), mime_type)
1465 class Node:
1466 """ A convenience wrapper for the given node
1467 """
1468 def __init__(self, cl, nodeid, cache=1):
1469 self.__dict__['cl'] = cl
1470 self.__dict__['nodeid'] = nodeid
1471 def keys(self, protected=1):
1472 return self.cl.getprops(protected=protected).keys()
1473 def values(self, protected=1):
1474 l = []
1475 for name in self.cl.getprops(protected=protected).keys():
1476 l.append(self.cl.get(self.nodeid, name))
1477 return l
1478 def items(self, protected=1):
1479 l = []
1480 for name in self.cl.getprops(protected=protected).keys():
1481 l.append((name, self.cl.get(self.nodeid, name)))
1482 return l
1483 def has_key(self, name):
1484 return self.cl.getprops().has_key(name)
1485 def get(self, name, default=None):
1486 if self.has_key(name):
1487 return self[name]
1488 else:
1489 return default
1490 def __getattr__(self, name):
1491 if self.__dict__.has_key(name):
1492 return self.__dict__[name]
1493 try:
1494 return self.cl.get(self.nodeid, name)
1495 except KeyError, value:
1496 # we trap this but re-raise it as AttributeError - all other
1497 # exceptions should pass through untrapped
1498 pass
1499 # nope, no such attribute
1500 raise AttributeError, str(value)
1501 def __getitem__(self, name):
1502 return self.cl.get(self.nodeid, name)
1503 def __setattr__(self, name, value):
1504 try:
1505 return self.cl.set(self.nodeid, **{name: value})
1506 except KeyError, value:
1507 raise AttributeError, str(value)
1508 def __setitem__(self, name, value):
1509 self.cl.set(self.nodeid, **{name: value})
1510 def history(self):
1511 return self.cl.history(self.nodeid)
1512 def retire(self):
1513 return self.cl.retire(self.nodeid)
1516 def Choice(name, db, *options):
1517 """Quick helper to create a simple class with choices
1518 """
1519 cl = Class(db, name, name=String(), order=String())
1520 for i in range(len(options)):
1521 cl.create(name=options[i], order=i)
1522 return Link(name)
1524 # vim: set filetype=python sts=4 sw=4 et si :