1 #
2 # Copyright (c) 2001 Bizar Software Pty Ltd (http://www.bizarsoftware.com.au/)
3 # This module is free software, and you may redistribute it and/or modify
4 # under the same terms as Python, so long as this copyright message and
5 # disclaimer are retained in their original form.
6 #
7 # IN NO EVENT SHALL BIZAR SOFTWARE PTY LTD BE LIABLE TO ANY PARTY FOR
8 # DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING
9 # OUT OF THE USE OF THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE
10 # POSSIBILITY OF SUCH DAMAGE.
11 #
12 # BIZAR SOFTWARE PTY LTD SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
13 # BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
14 # FOR A PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS"
15 # BASIS, AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE,
16 # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
17 #
18 #$Id: blobfiles.py,v 1.6 2002-07-09 03:02:52 richard Exp $
19 '''
20 This module exports file storage for roundup backends.
21 Files are stored into a directory hierarchy.
22 '''
24 import os
26 def files_in_dir(dir):
27 if not os.path.exists(dir):
28 return 0
29 num_files = 0
30 for dir_entry in os.listdir(dir):
31 full_filename = os.path.join(dir,dir_entry)
32 if os.path.isfile(full_filename):
33 num_files = num_files + 1
34 elif os.path.isdir(full_filename):
35 num_files = num_files + files_in_dir(full_filename)
36 return num_files
38 class FileStorage:
39 """Store files in some directory structure"""
40 # TODO: maybe set "files"
41 # def __init__(self):
42 # pass
44 def filename(self, classname, nodeid, property=None):
45 '''Determine what the filename for the given node and optionally
46 property is.
47 '''
48 if property:
49 name = '%s%s.%s'%(classname, nodeid, property)
50 else:
51 # roundupdb.FileClass never specified the property name, so don't
52 # include it
53 name = '%s%s'%(classname, nodeid)
55 # have a separate subdir for every thousand messages
56 subdir = str(int(nodeid) / 1000)
57 return os.path.join(self.dir, 'files', classname, subdir, name)
59 def filename_flat(self, classname, nodeid, property=None):
60 '''Determine what the filename for the given node and optionally
61 property is.
62 '''
63 if property:
64 return os.path.join(self.dir, 'files', '%s%s.%s'%(classname,
65 nodeid, property))
66 else:
67 # roundupdb.FileClass never specified the property name, so don't
68 # include it
69 return os.path.join(self.dir, 'files', '%s%s'%(classname,
70 nodeid))
72 def storefile(self, classname, nodeid, property, content):
73 '''Store the content of the file in the database. The property may be
74 None, in which case the filename does not indicate which property
75 is being saved.
76 '''
77 # determine the name of the file to write to
78 name = self.filename(classname, nodeid, property)
80 # make sure the file storage dir exists
81 if not os.path.exists(os.path.dirname(name)):
82 os.makedirs(os.path.dirname(name))
84 # open the temp file for writing
85 open(name + '.tmp', 'wb').write(content)
87 # save off the commit action
88 self.transactions.append((self._doStoreFile, (classname, nodeid,
89 property)))
91 def getfile(self, classname, nodeid, property):
92 '''Get the content of the file in the database.
93 '''
94 filename = self.filename(classname, nodeid, property)
95 try:
96 return open(filename, 'rb').read()
97 except:
98 # now try the temp pre-commit filename
99 try:
100 return open(filename+'.tmp', 'rb').read()
101 except:
102 # fallback to flat file storage
103 filename = self.filename_flat(classname, nodeid, property)
104 return open(filename, 'rb').read()
106 def numfiles(self):
107 '''Get number of files in storage, even across subdirectories.
108 '''
109 files_dir = os.path.join(self.dir, 'files')
110 return files_in_dir(files_dir)
112 def _doStoreFile(self, classname, nodeid, property, **databases):
113 '''Store the file as part of a transaction commit.
114 '''
115 # determine the name of the file to write to
116 name = self.filename(classname, nodeid, property)
118 # the file is currently ".tmp" - move it to its real name to commit
119 os.rename(name+".tmp", name)
121 # return the classname, nodeid so we reindex this content
122 return (classname, nodeid)
124 def _rollbackStoreFile(self, classname, nodeid, property, **databases):
125 '''Remove the temp file as a part of a rollback
126 '''
127 # determine the name of the file to delete
128 name = self.filename(classname, nodeid, property)
129 if os.path.exists(name+".tmp"):
130 os.remove(name+".tmp")
132 # $Log: not supported by cvs2svn $
133 # Revision 1.5 2002/07/08 06:58:15 richard
134 # cleaned up the indexer code:
135 # - it splits more words out (much simpler, faster splitter)
136 # - removed code we'll never use (roundup.roundup_indexer has the full
137 # implementation, and replaces roundup.indexer)
138 # - only index text/plain and rfc822/message (ideas for other text formats to
139 # index are welcome)
140 # - added simple unit test for indexer. Needs more tests for regression.
141 #
142 # Revision 1.4 2002/06/19 03:07:19 richard
143 # Moved the file storage commit into blobfiles where it belongs.
144 #
145 # Revision 1.3 2002/02/27 07:33:34 grubert
146 # . add, vim line and cvs log key.
147 #
148 #
149 # vim: set filetype=python ts=4 sw=4 et si