1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
19 """Hyperdatabase implementation, especially field types.
20 """
21 __docformat__ = 'restructuredtext'
23 # standard python modules
24 import os, re, shutil, weakref
25 # Python 2.3 ... 2.6 compatibility:
26 from roundup.anypy.sets_ import set
28 # roundup modules
29 import date, password
30 from support import ensureParentsExist, PrioList, sorted, reversed
31 from roundup.i18n import _
33 #
34 # Types
35 #
36 class _Type(object):
37 """A roundup property type."""
38 def __init__(self, required=False):
39 self.required = required
40 def __repr__(self):
41 ' more useful for dumps '
42 return '<%s.%s>'%(self.__class__.__module__, self.__class__.__name__)
43 def sort_repr (self, cls, val, name):
44 """Representation used for sorting. This should be a python
45 built-in type, otherwise sorting will take ages. Note that
46 individual backends may chose to use something different for
47 sorting as long as the outcome is the same.
48 """
49 return val
51 class String(_Type):
52 """An object designating a String property."""
53 def __init__(self, indexme='no', required=False):
54 super(String, self).__init__(required)
55 self.indexme = indexme == 'yes'
56 def from_raw(self, value, propname='', **kw):
57 """fix the CRLF/CR -> LF stuff"""
58 if propname == 'content':
59 # Why oh why wasn't the FileClass content property a File
60 # type from the beginning?
61 return value
62 return fixNewlines(value)
63 def sort_repr (self, cls, val, name):
64 if not val:
65 return val
66 if name == 'id':
67 return int(val)
68 return val.lower()
70 class Password(_Type):
71 """An object designating a Password property."""
72 def from_raw(self, value, **kw):
73 if not value:
74 return None
75 m = password.Password.pwre.match(value)
76 if m:
77 # password is being given to us encrypted
78 p = password.Password()
79 p.scheme = m.group(1)
80 if p.scheme not in 'SHA crypt plaintext'.split():
81 raise HyperdbValueError, \
82 ('property %s: unknown encryption scheme %r') %\
83 (kw['propname'], p.scheme)
84 p.password = m.group(2)
85 value = p
86 else:
87 try:
88 value = password.Password(value)
89 except password.PasswordValueError, message:
90 raise HyperdbValueError, \
91 _('property %s: %s')%(kw['propname'], message)
92 return value
93 def sort_repr (self, cls, val, name):
94 if not val:
95 return val
96 return str(val)
98 class Date(_Type):
99 """An object designating a Date property."""
100 def __init__(self, offset=None, required=False):
101 super(Date, self).__init__(required)
102 self._offset = offset
103 def offset(self, db):
104 if self._offset is not None:
105 return self._offset
106 return db.getUserTimezone()
107 def from_raw(self, value, db, **kw):
108 try:
109 value = date.Date(value, self.offset(db))
110 except ValueError, message:
111 raise HyperdbValueError, _('property %s: %r is an invalid '\
112 'date (%s)')%(kw['propname'], value, message)
113 return value
114 def range_from_raw(self, value, db):
115 """return Range value from given raw value with offset correction"""
116 return date.Range(value, date.Date, offset=self.offset(db))
117 def sort_repr (self, cls, val, name):
118 if not val:
119 return val
120 return str(val)
122 class Interval(_Type):
123 """An object designating an Interval property."""
124 def from_raw(self, value, **kw):
125 try:
126 value = date.Interval(value)
127 except ValueError, message:
128 raise HyperdbValueError, _('property %s: %r is an invalid '\
129 'date interval (%s)')%(kw['propname'], value, message)
130 return value
131 def sort_repr (self, cls, val, name):
132 if not val:
133 return val
134 return val.as_seconds()
136 class _Pointer(_Type):
137 """An object designating a Pointer property that links or multilinks
138 to a node in a specified class."""
139 def __init__(self, classname, do_journal='yes', required=False):
140 """ Default is to journal link and unlink events
141 """
142 super(_Pointer, self).__init__(required)
143 self.classname = classname
144 self.do_journal = do_journal == 'yes'
145 def __repr__(self):
146 """more useful for dumps. But beware: This is also used in schema
147 storage in SQL backends!
148 """
149 return '<%s.%s to "%s">'%(self.__class__.__module__,
150 self.__class__.__name__, self.classname)
152 class Link(_Pointer):
153 """An object designating a Link property that links to a
154 node in a specified class."""
155 def from_raw(self, value, db, propname, **kw):
156 if value == '-1' or not value:
157 value = None
158 else:
159 value = convertLinkValue(db, propname, self, value)
160 return value
161 def sort_repr (self, cls, val, name):
162 if not val:
163 return val
164 op = cls.labelprop()
165 if op == 'id':
166 return int(cls.get(val, op))
167 return cls.get(val, op)
169 class Multilink(_Pointer):
170 """An object designating a Multilink property that links
171 to nodes in a specified class.
173 "classname" indicates the class to link to
175 "do_journal" indicates whether the linked-to nodes should have
176 'link' and 'unlink' events placed in their journal
177 """
178 def from_raw(self, value, db, klass, propname, itemid, **kw):
179 if not value:
180 return []
182 # get the current item value if it's not a new item
183 if itemid and not itemid.startswith('-'):
184 curvalue = klass.get(itemid, propname)
185 else:
186 curvalue = []
188 # if the value is a comma-separated string then split it now
189 if isinstance(value, type('')):
190 value = value.split(',')
192 # handle each add/remove in turn
193 # keep an extra list for all items that are
194 # definitely in the new list (in case of e.g.
195 # <propname>=A,+B, which should replace the old
196 # list with A,B)
197 do_set = 1
198 newvalue = []
199 for item in value:
200 item = item.strip()
202 # skip blanks
203 if not item: continue
205 # handle +/-
206 remove = 0
207 if item.startswith('-'):
208 remove = 1
209 item = item[1:]
210 do_set = 0
211 elif item.startswith('+'):
212 item = item[1:]
213 do_set = 0
215 # look up the value
216 itemid = convertLinkValue(db, propname, self, item)
218 # perform the add/remove
219 if remove:
220 try:
221 curvalue.remove(itemid)
222 except ValueError:
223 raise HyperdbValueError, _('property %s: %r is not ' \
224 'currently an element')%(propname, item)
225 else:
226 newvalue.append(itemid)
227 if itemid not in curvalue:
228 curvalue.append(itemid)
230 # that's it, set the new Multilink property value,
231 # or overwrite it completely
232 if do_set:
233 value = newvalue
234 else:
235 value = curvalue
237 # TODO: one day, we'll switch to numeric ids and this will be
238 # unnecessary :(
239 value = [int(x) for x in value]
240 value.sort()
241 value = [str(x) for x in value]
242 return value
244 def sort_repr (self, cls, val, name):
245 if not val:
246 return val
247 op = cls.labelprop()
248 if op == 'id':
249 return [int(cls.get(v, op)) for v in val]
250 return [cls.get(v, op) for v in val]
252 class Boolean(_Type):
253 """An object designating a boolean property"""
254 def from_raw(self, value, **kw):
255 value = value.strip()
256 # checked is a common HTML checkbox value
257 value = value.lower() in ('checked', 'yes', 'true', 'on', '1')
258 return value
260 class Number(_Type):
261 """An object designating a numeric property"""
262 def from_raw(self, value, **kw):
263 value = value.strip()
264 try:
265 value = float(value)
266 except ValueError:
267 raise HyperdbValueError, _('property %s: %r is not a number')%(
268 kw['propname'], value)
269 return value
270 #
271 # Support for splitting designators
272 #
273 class DesignatorError(ValueError):
274 pass
275 def splitDesignator(designator, dre=re.compile(r'([^\d]+)(\d+)')):
276 """ Take a foo123 and return ('foo', 123)
277 """
278 m = dre.match(designator)
279 if m is None:
280 raise DesignatorError, _('"%s" not a node designator')%designator
281 return m.group(1), m.group(2)
283 class Proptree(object):
284 """ Simple tree data structure for optimizing searching of
285 properties. Each node in the tree represents a roundup Class
286 Property that has to be navigated for finding the given search
287 or sort properties. The need_for attribute is used for
288 distinguishing nodes in the tree used for sorting, searching or
289 retrieval: The attribute is a dictionary containing one or several
290 of the values 'sort', 'search', 'retrieve'.
292 The Proptree is also used for transitively searching attributes for
293 backends that do not support transitive search (e.g. anydbm). The
294 _val attribute with set_val is used for this.
295 """
297 def __init__(self, db, cls, name, props, parent=None, retr=False):
298 self.db = db
299 self.name = name
300 self.props = props
301 self.parent = parent
302 self._val = None
303 self.has_values = False
304 self.cls = cls
305 self.classname = None
306 self.uniqname = None
307 self.children = []
308 self.sortattr = []
309 self.propdict = {}
310 self.need_for = {'search' : True}
311 self.sort_direction = None
312 self.sort_ids = None
313 self.sort_ids_needed = False
314 self.sort_result = None
315 self.attr_sort_done = False
316 self.tree_sort_done = False
317 self.propclass = None
318 self.orderby = []
319 self.sql_idx = None # index of retrieved column in sql result
320 if parent:
321 self.root = parent.root
322 self.depth = parent.depth + 1
323 else:
324 self.root = self
325 self.seqno = 1
326 self.depth = 0
327 self.need_for['sort'] = True
328 self.id = self.root.seqno
329 self.root.seqno += 1
330 if self.cls:
331 self.classname = self.cls.classname
332 self.uniqname = '%s%s' % (self.cls.classname, self.id)
333 if not self.parent:
334 self.uniqname = self.cls.classname
335 if retr:
336 self.append_retr_props()
338 def append(self, name, need_for='search', retr=False):
339 """Append a property to self.children. Will create a new
340 propclass for the child.
341 """
342 if name in self.propdict:
343 pt = self.propdict[name]
344 pt.need_for[need_for] = True
345 if retr and isinstance(pt.propclass, Link):
346 pt.append_retr_props()
347 return pt
348 propclass = self.props[name]
349 cls = None
350 props = None
351 if isinstance(propclass, (Link, Multilink)):
352 cls = self.db.getclass(propclass.classname)
353 props = cls.getprops()
354 child = self.__class__(self.db, cls, name, props, parent = self)
355 child.need_for = {need_for : True}
356 child.propclass = propclass
357 self.children.append(child)
358 self.propdict[name] = child
359 if retr and isinstance(child.propclass, Link):
360 child.append_retr_props()
361 return child
363 def append_retr_props(self):
364 """Append properties for retrieval."""
365 for name, prop in self.cls.getprops(protected=1).iteritems():
366 if isinstance(prop, Multilink):
367 continue
368 self.append(name, need_for='retrieve')
370 def compute_sort_done(self, mlseen=False):
371 """ Recursively check if attribute is needed for sorting
372 ('sort' in self.need_for) or all children have tree_sort_done set and
373 sort_ids_needed unset: set self.tree_sort_done if one of the conditions
374 holds. Also remove sort_ids_needed recursively once having seen a
375 Multilink.
376 """
377 if isinstance (self.propclass, Multilink):
378 mlseen = True
379 if mlseen:
380 self.sort_ids_needed = False
381 self.tree_sort_done = True
382 for p in self.children:
383 p.compute_sort_done(mlseen)
384 if not p.tree_sort_done:
385 self.tree_sort_done = False
386 if 'sort' not in self.need_for:
387 self.tree_sort_done = True
388 if mlseen:
389 self.tree_sort_done = False
391 def ancestors(self):
392 p = self
393 while p.parent:
394 yield p
395 p = p.parent
397 def search(self, search_matches=None, sort=True):
398 """ Recursively search for the given properties in a proptree.
399 Once all properties are non-transitive, the search generates a
400 simple _filter call which does the real work
401 """
402 filterspec = {}
403 for p in self.children:
404 if 'search' in p.need_for:
405 if p.children:
406 p.search(sort = False)
407 filterspec[p.name] = p.val
408 self.val = self.cls._filter(search_matches, filterspec, sort and self)
409 return self.val
411 def sort (self, ids=None):
412 """ Sort ids by the order information stored in self. With
413 optimisations: Some order attributes may be precomputed (by the
414 backend) and some properties may already be sorted.
415 """
416 if ids is None:
417 ids = self.val
418 if self.sortattr and [s for s in self.sortattr if not s.attr_sort_done]:
419 return self._searchsort(ids, True, True)
420 return ids
422 def sortable_children(self, intermediate=False):
423 """ All children needed for sorting. If intermediate is True,
424 intermediate nodes (not being a sort attribute) are returned,
425 too.
426 """
427 return [p for p in self.children
428 if 'sort' in p.need_for and (intermediate or p.sort_direction)]
430 def __iter__(self):
431 """ Yield nodes in depth-first order -- visited nodes first """
432 for p in self.children:
433 yield p
434 for c in p:
435 yield c
437 def _get (self, ids):
438 """Lookup given ids -- possibly a list of list. We recurse until
439 we have a list of ids.
440 """
441 if not ids:
442 return ids
443 if isinstance (ids[0], list):
444 cids = [self._get(i) for i in ids]
445 else:
446 cids = [i and self.parent.cls.get(i, self.name) for i in ids]
447 if self.sortattr:
448 cids = [self._searchsort(i, False, True) for i in cids]
449 return cids
451 def _searchsort(self, ids=None, update=True, dosort=True):
452 """ Recursively compute the sort attributes. Note that ids
453 may be a deeply nested list of lists of ids if several
454 multilinks are encountered on the way from the root to an
455 individual attribute. We make sure that everything is properly
456 sorted on the way up. Note that the individual backend may
457 already have precomputed self.result or self.sort_ids. In this
458 case we do nothing for existing sa.result and recurse further if
459 self.sort_ids is available.
461 Yech, Multilinks: This gets especially complicated if somebody
462 sorts by different attributes of the same multilink (or
463 transitively across several multilinks). My use-case is sorting
464 by issue.messages.author and (reverse) by issue.messages.date.
465 In this case we sort the messages by author and date and use
466 this sorted list twice for sorting issues. This means that
467 issues are sorted by author and then by the time of the messages
468 *of this author*. Probably what the user intends in that case,
469 so we do *not* use two sorted lists of messages, one sorted by
470 author and one sorted by date for sorting issues.
471 """
472 for pt in self.sortable_children(intermediate = True):
473 # ids can be an empty list
474 if pt.tree_sort_done or not ids:
475 continue
476 if pt.sort_ids: # cached or computed by backend
477 cids = pt.sort_ids
478 else:
479 cids = pt._get(ids)
480 if pt.sort_direction and not pt.sort_result:
481 sortrep = pt.propclass.sort_repr
482 pt.sort_result = pt._sort_repr(sortrep, cids)
483 pt.sort_ids = cids
484 if pt.children:
485 pt._searchsort(cids, update, False)
486 if self.sortattr and dosort:
487 ids = self._sort(ids)
488 if not update:
489 for pt in self.sortable_children(intermediate = True):
490 pt.sort_ids = None
491 for pt in self.sortattr:
492 pt.sort_result = None
493 return ids
495 def _set_val(self, val):
496 """Check if self._val is already defined. If yes, we compute the
497 intersection of the old and the new value(s)
498 """
499 if self.has_values:
500 v = self._val
501 if not isinstance(self._val, type([])):
502 v = [self._val]
503 vals = set(v)
504 vals.intersection_update(val)
505 self._val = [v for v in vals]
506 else:
507 self._val = val
508 self.has_values = True
510 val = property(lambda self: self._val, _set_val)
512 def _sort(self, val):
513 """Finally sort by the given sortattr.sort_result. Note that we
514 do not sort by attrs having attr_sort_done set. The caller is
515 responsible for setting attr_sort_done only for trailing
516 attributes (otherwise the sort order is wrong). Since pythons
517 sort is stable, we can sort already sorted lists without
518 destroying the sort-order for items that compare equal with the
519 current sort.
521 Sorting-Strategy: We sort repeatedly by different sort-keys from
522 right to left. Since pythons sort is stable, we can safely do
523 that. An optimisation is a "run-length encoding" of the
524 sort-directions: If several sort attributes sort in the same
525 direction we can combine them into a single sort. Note that
526 repeated sorting is probably more efficient than using
527 compare-methods in python due to the overhead added by compare
528 methods.
529 """
530 if not val:
531 return val
532 sortattr = []
533 directions = []
534 dir_idx = []
535 idx = 0
536 curdir = None
537 for sa in self.sortattr:
538 if sa.attr_sort_done:
539 break
540 if sortattr:
541 assert len(sortattr[0]) == len(sa.sort_result)
542 sortattr.append (sa.sort_result)
543 if curdir != sa.sort_direction:
544 dir_idx.append (idx)
545 directions.append (sa.sort_direction)
546 curdir = sa.sort_direction
547 idx += 1
548 sortattr.append (val)
549 sortattr = zip (*sortattr)
550 for dir, i in reversed(zip(directions, dir_idx)):
551 rev = dir == '-'
552 sortattr = sorted (sortattr, key = lambda x:x[i:idx], reverse = rev)
553 idx = i
554 return [x[-1] for x in sortattr]
556 def _sort_repr(self, sortrep, ids):
557 """Call sortrep for given ids -- possibly a list of list. We
558 recurse until we have a list of ids.
559 """
560 if not ids:
561 return ids
562 if isinstance (ids[0], list):
563 res = [self._sort_repr(sortrep, i) for i in ids]
564 else:
565 res = [sortrep(self.cls, i, self.name) for i in ids]
566 return res
568 def __repr__(self):
569 r = ["proptree:" + self.name]
570 for n in self:
571 r.append("proptree:" + " " * n.depth + n.name)
572 return '\n'.join(r)
573 __str__ = __repr__
575 #
576 # the base Database class
577 #
578 class DatabaseError(ValueError):
579 """Error to be raised when there is some problem in the database code
580 """
581 pass
582 class Database:
583 """A database for storing records containing flexible data types.
585 This class defines a hyperdatabase storage layer, which the Classes use to
586 store their data.
589 Transactions
590 ------------
591 The Database should support transactions through the commit() and
592 rollback() methods. All other Database methods should be transaction-aware,
593 using data from the current transaction before looking up the database.
595 An implementation must provide an override for the get() method so that the
596 in-database value is returned in preference to the in-transaction value.
597 This is necessary to determine if any values have changed during a
598 transaction.
601 Implementation
602 --------------
604 All methods except __repr__ must be implemented by a concrete backend Database.
606 """
608 # flag to set on retired entries
609 RETIRED_FLAG = '__hyperdb_retired'
611 BACKEND_MISSING_STRING = None
612 BACKEND_MISSING_NUMBER = None
613 BACKEND_MISSING_BOOLEAN = None
615 def __init__(self, config, journaltag=None):
616 """Open a hyperdatabase given a specifier to some storage.
618 The 'storagelocator' is obtained from config.DATABASE.
619 The meaning of 'storagelocator' depends on the particular
620 implementation of the hyperdatabase. It could be a file name,
621 a directory path, a socket descriptor for a connection to a
622 database over the network, etc.
624 The 'journaltag' is a token that will be attached to the journal
625 entries for any edits done on the database. If 'journaltag' is
626 None, the database is opened in read-only mode: the Class.create(),
627 Class.set(), and Class.retire() methods are disabled.
628 """
629 raise NotImplementedError
631 def post_init(self):
632 """Called once the schema initialisation has finished.
633 If 'refresh' is true, we want to rebuild the backend
634 structures.
635 """
636 raise NotImplementedError
638 def refresh_database(self):
639 """Called to indicate that the backend should rebuild all tables
640 and structures. Not called in normal usage."""
641 raise NotImplementedError
643 def __getattr__(self, classname):
644 """A convenient way of calling self.getclass(classname)."""
645 raise NotImplementedError
647 def addclass(self, cl):
648 """Add a Class to the hyperdatabase.
649 """
650 raise NotImplementedError
652 def getclasses(self):
653 """Return a list of the names of all existing classes."""
654 raise NotImplementedError
656 def getclass(self, classname):
657 """Get the Class object representing a particular class.
659 If 'classname' is not a valid class name, a KeyError is raised.
660 """
661 raise NotImplementedError
663 def clear(self):
664 """Delete all database contents.
665 """
666 raise NotImplementedError
668 def getclassdb(self, classname, mode='r'):
669 """Obtain a connection to the class db that will be used for
670 multiple actions.
671 """
672 raise NotImplementedError
674 def addnode(self, classname, nodeid, node):
675 """Add the specified node to its class's db.
676 """
677 raise NotImplementedError
679 def serialise(self, classname, node):
680 """Copy the node contents, converting non-marshallable data into
681 marshallable data.
682 """
683 return node
685 def setnode(self, classname, nodeid, node):
686 """Change the specified node.
687 """
688 raise NotImplementedError
690 def unserialise(self, classname, node):
691 """Decode the marshalled node data
692 """
693 return node
695 def getnode(self, classname, nodeid):
696 """Get a node from the database.
698 'cache' exists for backwards compatibility, and is not used.
699 """
700 raise NotImplementedError
702 def hasnode(self, classname, nodeid):
703 """Determine if the database has a given node.
704 """
705 raise NotImplementedError
707 def countnodes(self, classname):
708 """Count the number of nodes that exist for a particular Class.
709 """
710 raise NotImplementedError
712 def storefile(self, classname, nodeid, property, content):
713 """Store the content of the file in the database.
715 The property may be None, in which case the filename does not
716 indicate which property is being saved.
717 """
718 raise NotImplementedError
720 def getfile(self, classname, nodeid, property):
721 """Get the content of the file in the database.
722 """
723 raise NotImplementedError
725 def addjournal(self, classname, nodeid, action, params):
726 """ Journal the Action
727 'action' may be:
729 'create' or 'set' -- 'params' is a dictionary of property values
730 'link' or 'unlink' -- 'params' is (classname, nodeid, propname)
731 'retire' -- 'params' is None
732 """
733 raise NotImplementedError
735 def getjournal(self, classname, nodeid):
736 """ get the journal for id
737 """
738 raise NotImplementedError
740 def pack(self, pack_before):
741 """ pack the database
742 """
743 raise NotImplementedError
745 def commit(self):
746 """ Commit the current transactions.
748 Save all data changed since the database was opened or since the
749 last commit() or rollback().
751 fail_ok indicates that the commit is allowed to fail. This is used
752 in the web interface when committing cleaning of the session
753 database. We don't care if there's a concurrency issue there.
755 The only backend this seems to affect is postgres.
756 """
757 raise NotImplementedError
759 def rollback(self):
760 """ Reverse all actions from the current transaction.
762 Undo all the changes made since the database was opened or the last
763 commit() or rollback() was performed.
764 """
765 raise NotImplementedError
767 def close(self):
768 """Close the database.
770 This method must be called at the end of processing.
772 """
774 def iter_roles(roles):
775 ''' handle the text processing of turning the roles list
776 into something python can use more easily
777 '''
778 if not roles or not roles.strip():
779 raise StopIteration, "Empty roles given"
780 for role in [x.lower().strip() for x in roles.split(',')]:
781 yield role
784 #
785 # The base Class class
786 #
787 class Class:
788 """ The handle to a particular class of nodes in a hyperdatabase.
790 All methods except __repr__ and getnode must be implemented by a
791 concrete backend Class.
792 """
794 def __init__(self, db, classname, **properties):
795 """Create a new class with a given name and property specification.
797 'classname' must not collide with the name of an existing class,
798 or a ValueError is raised. The keyword arguments in 'properties'
799 must map names to property objects, or a TypeError is raised.
800 """
801 for name in 'creation activity creator actor'.split():
802 if properties.has_key(name):
803 raise ValueError, '"creation", "activity", "creator" and '\
804 '"actor" are reserved'
806 self.classname = classname
807 self.properties = properties
808 self.db = weakref.proxy(db) # use a weak ref to avoid circularity
809 self.key = ''
811 # should we journal changes (default yes)
812 self.do_journal = 1
814 # do the db-related init stuff
815 db.addclass(self)
817 actions = "create set retire restore".split()
818 self.auditors = dict([(a, PrioList()) for a in actions])
819 self.reactors = dict([(a, PrioList()) for a in actions])
821 def __repr__(self):
822 """Slightly more useful representation
823 """
824 return '<hyperdb.Class "%s">'%self.classname
826 # Editing nodes:
828 def create(self, **propvalues):
829 """Create a new node of this class and return its id.
831 The keyword arguments in 'propvalues' map property names to values.
833 The values of arguments must be acceptable for the types of their
834 corresponding properties or a TypeError is raised.
836 If this class has a key property, it must be present and its value
837 must not collide with other key strings or a ValueError is raised.
839 Any other properties on this class that are missing from the
840 'propvalues' dictionary are set to None.
842 If an id in a link or multilink property does not refer to a valid
843 node, an IndexError is raised.
844 """
845 raise NotImplementedError
847 _marker = []
848 def get(self, nodeid, propname, default=_marker, cache=1):
849 """Get the value of a property on an existing node of this class.
851 'nodeid' must be the id of an existing node of this class or an
852 IndexError is raised. 'propname' must be the name of a property
853 of this class or a KeyError is raised.
855 'cache' exists for backwards compatibility, and is not used.
856 """
857 raise NotImplementedError
859 # not in spec
860 def getnode(self, nodeid):
861 """ Return a convenience wrapper for the node.
863 'nodeid' must be the id of an existing node of this class or an
864 IndexError is raised.
866 'cache' exists for backwards compatibility, and is not used.
867 """
868 return Node(self, nodeid)
870 def getnodeids(self, retired=None):
871 """Retrieve all the ids of the nodes for a particular Class.
872 """
873 raise NotImplementedError
875 def set(self, nodeid, **propvalues):
876 """Modify a property on an existing node of this class.
878 'nodeid' must be the id of an existing node of this class or an
879 IndexError is raised.
881 Each key in 'propvalues' must be the name of a property of this
882 class or a KeyError is raised.
884 All values in 'propvalues' must be acceptable types for their
885 corresponding properties or a TypeError is raised.
887 If the value of the key property is set, it must not collide with
888 other key strings or a ValueError is raised.
890 If the value of a Link or Multilink property contains an invalid
891 node id, a ValueError is raised.
892 """
893 raise NotImplementedError
895 def retire(self, nodeid):
896 """Retire a node.
898 The properties on the node remain available from the get() method,
899 and the node's id is never reused.
901 Retired nodes are not returned by the find(), list(), or lookup()
902 methods, and other nodes may reuse the values of their key properties.
903 """
904 raise NotImplementedError
906 def restore(self, nodeid):
907 """Restpre a retired node.
909 Make node available for all operations like it was before retirement.
910 """
911 raise NotImplementedError
913 def is_retired(self, nodeid):
914 """Return true if the node is rerired
915 """
916 raise NotImplementedError
918 def destroy(self, nodeid):
919 """Destroy a node.
921 WARNING: this method should never be used except in extremely rare
922 situations where there could never be links to the node being
923 deleted
925 WARNING: use retire() instead
927 WARNING: the properties of this node will not be available ever again
929 WARNING: really, use retire() instead
931 Well, I think that's enough warnings. This method exists mostly to
932 support the session storage of the cgi interface.
934 The node is completely removed from the hyperdb, including all journal
935 entries. It will no longer be available, and will generally break code
936 if there are any references to the node.
937 """
939 def history(self, nodeid):
940 """Retrieve the journal of edits on a particular node.
942 'nodeid' must be the id of an existing node of this class or an
943 IndexError is raised.
945 The returned list contains tuples of the form
947 (date, tag, action, params)
949 'date' is a Timestamp object specifying the time of the change and
950 'tag' is the journaltag specified when the database was opened.
951 """
952 raise NotImplementedError
954 # Locating nodes:
955 def hasnode(self, nodeid):
956 """Determine if the given nodeid actually exists
957 """
958 raise NotImplementedError
960 def setkey(self, propname):
961 """Select a String property of this class to be the key property.
963 'propname' must be the name of a String property of this class or
964 None, or a TypeError is raised. The values of the key property on
965 all existing nodes must be unique or a ValueError is raised.
966 """
967 raise NotImplementedError
969 def setlabelprop(self, labelprop):
970 """Set the label property. Used for override of labelprop
971 resolution order.
972 """
973 if labelprop not in self.getprops():
974 raise ValueError, _("Not a property name: %s") % labelprop
975 self._labelprop = labelprop
977 def setorderprop(self, orderprop):
978 """Set the order property. Used for override of orderprop
979 resolution order
980 """
981 if orderprop not in self.getprops():
982 raise ValueError, _("Not a property name: %s") % orderprop
983 self._orderprop = orderprop
985 def getkey(self):
986 """Return the name of the key property for this class or None."""
987 raise NotImplementedError
989 def labelprop(self, default_to_id=0):
990 """Return the property name for a label for the given node.
992 This method attempts to generate a consistent label for the node.
993 It tries the following in order:
995 0. self._labelprop if set
996 1. key property
997 2. "name" property
998 3. "title" property
999 4. first property from the sorted property name list
1000 """
1001 if hasattr(self, '_labelprop'):
1002 return self._labelprop
1003 k = self.getkey()
1004 if k:
1005 return k
1006 props = self.getprops()
1007 if props.has_key('name'):
1008 return 'name'
1009 elif props.has_key('title'):
1010 return 'title'
1011 if default_to_id:
1012 return 'id'
1013 props = props.keys()
1014 props.sort()
1015 return props[0]
1017 def orderprop(self):
1018 """Return the property name to use for sorting for the given node.
1020 This method computes the property for sorting.
1021 It tries the following in order:
1023 0. self._orderprop if set
1024 1. "order" property
1025 2. self.labelprop()
1026 """
1028 if hasattr(self, '_orderprop'):
1029 return self._orderprop
1030 props = self.getprops()
1031 if props.has_key('order'):
1032 return 'order'
1033 return self.labelprop()
1035 def lookup(self, keyvalue):
1036 """Locate a particular node by its key property and return its id.
1038 If this class has no key property, a TypeError is raised. If the
1039 'keyvalue' matches one of the values for the key property among
1040 the nodes in this class, the matching node's id is returned;
1041 otherwise a KeyError is raised.
1042 """
1043 raise NotImplementedError
1045 def find(self, **propspec):
1046 """Get the ids of nodes in this class which link to the given nodes.
1048 'propspec' consists of keyword args propname={nodeid:1,}
1049 'propname' must be the name of a property in this class, or a
1050 KeyError is raised. That property must be a Link or Multilink
1051 property, or a TypeError is raised.
1053 Any node in this class whose 'propname' property links to any of the
1054 nodeids will be returned. Used by the full text indexing, which knows
1055 that "foo" occurs in msg1, msg3 and file7, so we have hits on these
1056 issues:
1058 db.issue.find(messages={'1':1,'3':1}, files={'7':1})
1059 """
1060 raise NotImplementedError
1062 def _filter(self, search_matches, filterspec, sort=(None,None),
1063 group=(None,None)):
1064 """For some backends this implements the non-transitive
1065 search, for more information see the filter method.
1066 """
1067 raise NotImplementedError
1069 def _proptree(self, filterspec, sortattr=[], retr=False):
1070 """Build a tree of all transitive properties in the given
1071 filterspec.
1072 If we retrieve (retr is True) linked items we don't follow
1073 across multilinks. We also don't follow if the searched value
1074 can contain NULL values.
1075 """
1076 proptree = Proptree(self.db, self, '', self.getprops(), retr=retr)
1077 for key, v in filterspec.iteritems():
1078 keys = key.split('.')
1079 p = proptree
1080 mlseen = False
1081 for k in keys:
1082 if isinstance (p.propclass, Multilink):
1083 mlseen = True
1084 isnull = v == '-1' or v is None
1085 nullin = isinstance(v, type([])) and ('-1' in v or None in v)
1086 r = retr and not mlseen and not isnull and not nullin
1087 p = p.append(k, retr=r)
1088 p.val = v
1089 multilinks = {}
1090 for s in sortattr:
1091 keys = s[1].split('.')
1092 p = proptree
1093 mlseen = False
1094 for k in keys:
1095 if isinstance (p.propclass, Multilink):
1096 mlseen = True
1097 r = retr and not mlseen
1098 p = p.append(k, need_for='sort', retr=r)
1099 if isinstance (p.propclass, Multilink):
1100 multilinks[p] = True
1101 if p.cls:
1102 p = p.append(p.cls.orderprop(), need_for='sort')
1103 if p.sort_direction: # if an orderprop is also specified explicitly
1104 continue
1105 p.sort_direction = s[0]
1106 proptree.sortattr.append (p)
1107 for p in multilinks.iterkeys():
1108 sattr = {}
1109 for c in p:
1110 if c.sort_direction:
1111 sattr [c] = True
1112 for sa in proptree.sortattr:
1113 if sa in sattr:
1114 p.sortattr.append (sa)
1115 return proptree
1117 def get_transitive_prop(self, propname_path, default = None):
1118 """Expand a transitive property (individual property names
1119 separated by '.' into a new property at the end of the path. If
1120 one of the names does not refer to a valid property, we return
1121 None.
1122 Example propname_path (for class issue): "messages.author"
1123 """
1124 props = self.db.getclass(self.classname).getprops()
1125 for k in propname_path.split('.'):
1126 try:
1127 prop = props[k]
1128 except (KeyError, TypeError):
1129 return default
1130 cl = getattr(prop, 'classname', None)
1131 props = None
1132 if cl:
1133 props = self.db.getclass(cl).getprops()
1134 return prop
1136 def _sortattr(self, sort=[], group=[]):
1137 """Build a single list of sort attributes in the correct order
1138 with sanity checks (no duplicate properties) included. Always
1139 sort last by id -- if id is not already in sortattr.
1140 """
1141 seen = {}
1142 sortattr = []
1143 for srt in group, sort:
1144 if not isinstance(srt, list):
1145 srt = [srt]
1146 for s in srt:
1147 if s[1] and s[1] not in seen:
1148 sortattr.append((s[0] or '+', s[1]))
1149 seen[s[1]] = True
1150 if 'id' not in seen :
1151 sortattr.append(('+', 'id'))
1152 return sortattr
1154 def filter(self, search_matches, filterspec, sort=[], group=[]):
1155 """Return a list of the ids of the active nodes in this class that
1156 match the 'filter' spec, sorted by the group spec and then the
1157 sort spec.
1159 "filterspec" is {propname: value(s)}
1161 "sort" and "group" are [(dir, prop), ...] where dir is '+', '-'
1162 or None and prop is a prop name or None. Note that for
1163 backward-compatibility reasons a single (dir, prop) tuple is
1164 also allowed.
1166 "search_matches" is a container type
1168 The filter must match all properties specificed. If the property
1169 value to match is a list:
1171 1. String properties must match all elements in the list, and
1172 2. Other properties must match any of the elements in the list.
1174 Note that now the propname in filterspec and prop in a
1175 sort/group spec may be transitive, i.e., it may contain
1176 properties of the form link.link.link.name, e.g. you can search
1177 for all issues where a message was added by a certain user in
1178 the last week with a filterspec of
1179 {'messages.author' : '42', 'messages.creation' : '.-1w;'}
1181 Implementation note:
1182 This implements a non-optimized version of Transitive search
1183 using _filter implemented in a backend class. A more efficient
1184 version can be implemented in the individual backends -- e.g.,
1185 an SQL backend will want to create a single SQL statement and
1186 override the filter method instead of implementing _filter.
1187 """
1188 sortattr = self._sortattr(sort = sort, group = group)
1189 proptree = self._proptree(filterspec, sortattr)
1190 proptree.search(search_matches)
1191 return proptree.sort()
1193 # non-optimized filter_iter, a backend may chose to implement a
1194 # better version that provides a real iterator that pre-fills the
1195 # cache for each id returned. Note that the filter_iter doesn't
1196 # promise to correctly sort by multilink (which isn't sane to do
1197 # anyway).
1198 filter_iter = filter
1200 def count(self):
1201 """Get the number of nodes in this class.
1203 If the returned integer is 'numnodes', the ids of all the nodes
1204 in this class run from 1 to numnodes, and numnodes+1 will be the
1205 id of the next node to be created in this class.
1206 """
1207 raise NotImplementedError
1209 # Manipulating properties:
1210 def getprops(self, protected=1):
1211 """Return a dictionary mapping property names to property objects.
1212 If the "protected" flag is true, we include protected properties -
1213 those which may not be modified.
1214 """
1215 raise NotImplementedError
1217 def get_required_props(self, propnames = []):
1218 """Return a dict of property names mapping to property objects.
1219 All properties that have the "required" flag set will be
1220 returned in addition to all properties in the propnames
1221 parameter.
1222 """
1223 props = self.getprops(protected = False)
1224 pdict = dict([(p, props[p]) for p in propnames])
1225 pdict.update([(k, v) for k, v in props.iteritems() if v.required])
1226 return pdict
1228 def addprop(self, **properties):
1229 """Add properties to this class.
1231 The keyword arguments in 'properties' must map names to property
1232 objects, or a TypeError is raised. None of the keys in 'properties'
1233 may collide with the names of existing properties, or a ValueError
1234 is raised before any properties have been added.
1235 """
1236 raise NotImplementedError
1238 def index(self, nodeid):
1239 """Add (or refresh) the node to search indexes"""
1240 raise NotImplementedError
1242 #
1243 # Detector interface
1244 #
1245 def audit(self, event, detector, priority = 100):
1246 """Register an auditor detector"""
1247 self.auditors[event].append((priority, detector.__name__, detector))
1249 def fireAuditors(self, event, nodeid, newvalues):
1250 """Fire all registered auditors"""
1251 for prio, name, audit in self.auditors[event]:
1252 audit(self.db, self, nodeid, newvalues)
1254 def react(self, event, detector, priority = 100):
1255 """Register a reactor detector"""
1256 self.reactors[event].append((priority, detector.__name__, detector))
1258 def fireReactors(self, event, nodeid, oldvalues):
1259 """Fire all registered reactors"""
1260 for prio, name, react in self.reactors[event]:
1261 react(self.db, self, nodeid, oldvalues)
1263 #
1264 # import / export support
1265 #
1266 def export_propnames(self):
1267 """List the property names for export from this Class"""
1268 propnames = self.getprops().keys()
1269 propnames.sort()
1270 return propnames
1272 def import_journals(self, entries):
1273 """Import a class's journal.
1275 Uses setjournal() to set the journal for each item.
1276 Strategy for import: Sort first by id, then import journals for
1277 each id, this way the memory footprint is a lot smaller than the
1278 initial implementation which stored everything in a big hash by
1279 id and then proceeded to import journals for each id."""
1280 properties = self.getprops()
1281 a = []
1282 for l in entries:
1283 # first element in sorted list is the (numeric) id
1284 # in python2.4 and up we would use sorted with a key...
1285 a.append ((int (l [0].strip ("'")), l))
1286 a.sort ()
1289 last = 0
1290 r = []
1291 for n, l in a:
1292 nodeid, jdate, user, action, params = map(eval, l)
1293 assert (str(n) == nodeid)
1294 if n != last:
1295 if r:
1296 self.db.setjournal(self.classname, str(last), r)
1297 last = n
1298 r = []
1300 if action == 'set':
1301 for propname, value in params.iteritems():
1302 prop = properties[propname]
1303 if value is None:
1304 pass
1305 elif isinstance(prop, Date):
1306 value = date.Date(value)
1307 elif isinstance(prop, Interval):
1308 value = date.Interval(value)
1309 elif isinstance(prop, Password):
1310 pwd = password.Password()
1311 pwd.unpack(value)
1312 value = pwd
1313 params[propname] = value
1314 elif action == 'create' and params:
1315 # old tracker with data stored in the create!
1316 params = {}
1317 r.append((nodeid, date.Date(jdate), user, action, params))
1318 if r:
1319 self.db.setjournal(self.classname, nodeid, r)
1321 #
1322 # convenience methods
1323 #
1324 def get_roles(self, nodeid):
1325 """Return iterator for all roles for this nodeid.
1327 Yields string-processed roles.
1328 This method can be overridden to provide a hook where we can
1329 insert other permission models (e.g. get roles from database)
1330 In standard schemas only a user has a roles property but
1331 this may be different in customized schemas.
1332 Note that this is the *central place* where role
1333 processing happens!
1334 """
1335 node = self.db.getnode(self.classname, nodeid)
1336 return iter_roles(node['roles'])
1338 def has_role(self, nodeid, *roles):
1339 '''See if this node has any roles that appear in roles.
1341 For convenience reasons we take a list.
1342 In standard schemas only a user has a roles property but
1343 this may be different in customized schemas.
1344 '''
1345 roles = dict.fromkeys ([r.strip().lower() for r in roles])
1346 for role in self.get_roles(nodeid):
1347 if role in roles:
1348 return True
1349 return False
1352 class HyperdbValueError(ValueError):
1353 """ Error converting a raw value into a Hyperdb value """
1354 pass
1356 def convertLinkValue(db, propname, prop, value, idre=re.compile('^\d+$')):
1357 """ Convert the link value (may be id or key value) to an id value. """
1358 linkcl = db.classes[prop.classname]
1359 if not idre.match(value):
1360 if linkcl.getkey():
1361 try:
1362 value = linkcl.lookup(value)
1363 except KeyError, message:
1364 raise HyperdbValueError, _('property %s: %r is not a %s.')%(
1365 propname, value, prop.classname)
1366 else:
1367 raise HyperdbValueError, _('you may only enter ID values '\
1368 'for property %s')%propname
1369 return value
1371 def fixNewlines(text):
1372 """ Homogenise line endings.
1374 Different web clients send different line ending values, but
1375 other systems (eg. email) don't necessarily handle those line
1376 endings. Our solution is to convert all line endings to LF.
1377 """
1378 text = text.replace('\r\n', '\n')
1379 return text.replace('\r', '\n')
1381 def rawToHyperdb(db, klass, itemid, propname, value, **kw):
1382 """ Convert the raw (user-input) value to a hyperdb-storable value. The
1383 value is for the "propname" property on itemid (may be None for a
1384 new item) of "klass" in "db".
1386 The value is usually a string, but in the case of multilink inputs
1387 it may be either a list of strings or a string with comma-separated
1388 values.
1389 """
1390 properties = klass.getprops()
1392 # ensure it's a valid property name
1393 propname = propname.strip()
1394 try:
1395 proptype = properties[propname]
1396 except KeyError:
1397 raise HyperdbValueError, _('%r is not a property of %s')%(propname,
1398 klass.classname)
1400 # if we got a string, strip it now
1401 if isinstance(value, type('')):
1402 value = value.strip()
1404 # convert the input value to a real property value
1405 value = proptype.from_raw(value, db=db, klass=klass,
1406 propname=propname, itemid=itemid, **kw)
1408 return value
1410 class FileClass:
1411 """ A class that requires the "content" property and stores it on
1412 disk.
1413 """
1414 default_mime_type = 'text/plain'
1416 def __init__(self, db, classname, **properties):
1417 """The newly-created class automatically includes the "content"
1418 property.
1419 """
1420 if not properties.has_key('content'):
1421 properties['content'] = String(indexme='yes')
1423 def export_propnames(self):
1424 """ Don't export the "content" property
1425 """
1426 propnames = self.getprops().keys()
1427 propnames.remove('content')
1428 propnames.sort()
1429 return propnames
1431 def exportFilename(self, dirname, nodeid):
1432 subdir_filename = self.db.subdirFilename(self.classname, nodeid)
1433 return os.path.join(dirname, self.classname+'-files', subdir_filename)
1435 def export_files(self, dirname, nodeid):
1436 """ Export the "content" property as a file, not csv column
1437 """
1438 source = self.db.filename(self.classname, nodeid)
1440 dest = self.exportFilename(dirname, nodeid)
1441 ensureParentsExist(dest)
1442 shutil.copyfile(source, dest)
1444 def import_files(self, dirname, nodeid):
1445 """ Import the "content" property as a file
1446 """
1447 source = self.exportFilename(dirname, nodeid)
1449 dest = self.db.filename(self.classname, nodeid, create=1)
1450 ensureParentsExist(dest)
1451 shutil.copyfile(source, dest)
1453 mime_type = None
1454 props = self.getprops()
1455 if props.has_key('type'):
1456 mime_type = self.get(nodeid, 'type')
1457 if not mime_type:
1458 mime_type = self.default_mime_type
1459 if props['content'].indexme:
1460 self.db.indexer.add_text((self.classname, nodeid, 'content'),
1461 self.get(nodeid, 'content'), mime_type)
1463 class Node:
1464 """ A convenience wrapper for the given node
1465 """
1466 def __init__(self, cl, nodeid, cache=1):
1467 self.__dict__['cl'] = cl
1468 self.__dict__['nodeid'] = nodeid
1469 def keys(self, protected=1):
1470 return self.cl.getprops(protected=protected).keys()
1471 def values(self, protected=1):
1472 l = []
1473 for name in self.cl.getprops(protected=protected).keys():
1474 l.append(self.cl.get(self.nodeid, name))
1475 return l
1476 def items(self, protected=1):
1477 l = []
1478 for name in self.cl.getprops(protected=protected).keys():
1479 l.append((name, self.cl.get(self.nodeid, name)))
1480 return l
1481 def has_key(self, name):
1482 return self.cl.getprops().has_key(name)
1483 def get(self, name, default=None):
1484 if self.has_key(name):
1485 return self[name]
1486 else:
1487 return default
1488 def __getattr__(self, name):
1489 if self.__dict__.has_key(name):
1490 return self.__dict__[name]
1491 try:
1492 return self.cl.get(self.nodeid, name)
1493 except KeyError, value:
1494 # we trap this but re-raise it as AttributeError - all other
1495 # exceptions should pass through untrapped
1496 pass
1497 # nope, no such attribute
1498 raise AttributeError, str(value)
1499 def __getitem__(self, name):
1500 return self.cl.get(self.nodeid, name)
1501 def __setattr__(self, name, value):
1502 try:
1503 return self.cl.set(self.nodeid, **{name: value})
1504 except KeyError, value:
1505 raise AttributeError, str(value)
1506 def __setitem__(self, name, value):
1507 self.cl.set(self.nodeid, **{name: value})
1508 def history(self):
1509 return self.cl.history(self.nodeid)
1510 def retire(self):
1511 return self.cl.retire(self.nodeid)
1514 def Choice(name, db, *options):
1515 """Quick helper to create a simple class with choices
1516 """
1517 cl = Class(db, name, name=String(), order=String())
1518 for i in range(len(options)):
1519 cl.create(name=options[i], order=i)
1520 return Link(name)
1522 # vim: set filetype=python sts=4 sw=4 et si :