1 """Shared support for scanning document type declarations in HTML and XHTML."""
3 import re, string
5 _declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
6 _declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
8 del re
11 class ParserBase:
12 """Parser base class which provides some common support methods used
13 by the SGML/HTML and XHTML parsers."""
15 def reset(self):
16 self.lineno = 1
17 self.offset = 0
19 def getpos(self):
20 """Return current line number and offset."""
21 return self.lineno, self.offset
23 def error(self, message):
24 """Return an error, showing current line number and offset.
26 Concrete subclasses *must* override this method.
27 """
28 raise NotImplementedError
30 # Internal -- update line number and offset. This should be
31 # called for each piece of data exactly once, in order -- in other
32 # words the concatenation of all the input strings to this
33 # function should be exactly the entire input.
34 def updatepos(self, i, j):
35 if i >= j:
36 return j
37 rawdata = self.rawdata
38 nlines = rawdata.count("\n", i, j)
39 if nlines:
40 self.lineno = self.lineno + nlines
41 pos = rawdata.rindex("\n", i, j) # Should not fail
42 self.offset = j-(pos+1)
43 else:
44 self.offset = self.offset + j-i
45 return j
47 _decl_otherchars = ''
49 # Internal -- parse declaration (for use by subclasses).
50 def parse_declaration(self, i):
51 # This is some sort of declaration; in "HTML as
52 # deployed," this should only be the document type
53 # declaration ("<!DOCTYPE html...>").
54 rawdata = self.rawdata
55 import sys
56 j = i + 2
57 assert rawdata[i:j] == "<!", "unexpected call to parse_declaration"
58 if rawdata[j:j+1] in ("-", ""):
59 # Start of comment followed by buffer boundary,
60 # or just a buffer boundary.
61 return -1
62 # in practice, this should look like: ((name|stringlit) S*)+ '>'
63 n = len(rawdata)
64 decltype, j = self._scan_name(j, i)
65 if j < 0:
66 return j
67 if decltype == "doctype":
68 self._decl_otherchars = ''
69 while j < n:
70 c = rawdata[j]
71 if c == ">":
72 # end of declaration syntax
73 data = rawdata[i+2:j]
74 if decltype == "doctype":
75 self.handle_decl(data)
76 else:
77 self.unknown_decl(data)
78 return j + 1
79 if c in "\"'":
80 m = _declstringlit_match(rawdata, j)
81 if not m:
82 return -1 # incomplete
83 j = m.end()
84 elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
85 name, j = self._scan_name(j, i)
86 elif c in self._decl_otherchars:
87 j = j + 1
88 elif c == "[":
89 if decltype == "doctype":
90 j = self._parse_doctype_subset(j + 1, i)
91 else:
92 self.error("unexpected '[' char in declaration")
93 else:
94 self.error(
95 "unexpected %s char in declaration" % `rawdata[j]`)
96 if j < 0:
97 return j
98 return -1 # incomplete
100 # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
101 # returning the index just past any whitespace following the trailing ']'.
102 def _parse_doctype_subset(self, i, declstartpos):
103 rawdata = self.rawdata
104 n = len(rawdata)
105 j = i
106 while j < n:
107 c = rawdata[j]
108 if c == "<":
109 s = rawdata[j:j+2]
110 if s == "<":
111 # end of buffer; incomplete
112 return -1
113 if s != "<!":
114 self.updatepos(declstartpos, j + 1)
115 self.error("unexpected char in internal subset (in %s)"
116 % `s`)
117 if (j + 2) == n:
118 # end of buffer; incomplete
119 return -1
120 if (j + 4) > n:
121 # end of buffer; incomplete
122 return -1
123 if rawdata[j:j+4] == "<!--":
124 j = self.parse_comment(j, report=0)
125 if j < 0:
126 return j
127 continue
128 name, j = self._scan_name(j + 2, declstartpos)
129 if j == -1:
130 return -1
131 if name not in ("attlist", "element", "entity", "notation"):
132 self.updatepos(declstartpos, j + 2)
133 self.error(
134 "unknown declaration %s in internal subset" % `name`)
135 # handle the individual names
136 meth = getattr(self, "_parse_doctype_" + name)
137 j = meth(j, declstartpos)
138 if j < 0:
139 return j
140 elif c == "%":
141 # parameter entity reference
142 if (j + 1) == n:
143 # end of buffer; incomplete
144 return -1
145 s, j = self._scan_name(j + 1, declstartpos)
146 if j < 0:
147 return j
148 if rawdata[j] == ";":
149 j = j + 1
150 elif c == "]":
151 j = j + 1
152 while j < n and rawdata[j] in string.whitespace:
153 j = j + 1
154 if j < n:
155 if rawdata[j] == ">":
156 return j
157 self.updatepos(declstartpos, j)
158 self.error("unexpected char after internal subset")
159 else:
160 return -1
161 elif c in string.whitespace:
162 j = j + 1
163 else:
164 self.updatepos(declstartpos, j)
165 self.error("unexpected char %s in internal subset" % `c`)
166 # end of buffer reached
167 return -1
169 # Internal -- scan past <!ELEMENT declarations
170 def _parse_doctype_element(self, i, declstartpos):
171 rawdata = self.rawdata
172 n = len(rawdata)
173 name, j = self._scan_name(i, declstartpos)
174 if j == -1:
175 return -1
176 # style content model; just skip until '>'
177 if '>' in rawdata[j:]:
178 return rawdata.find(">", j) + 1
179 return -1
181 # Internal -- scan past <!ATTLIST declarations
182 def _parse_doctype_attlist(self, i, declstartpos):
183 rawdata = self.rawdata
184 name, j = self._scan_name(i, declstartpos)
185 c = rawdata[j:j+1]
186 if c == "":
187 return -1
188 if c == ">":
189 return j + 1
190 while 1:
191 # scan a series of attribute descriptions; simplified:
192 # name type [value] [#constraint]
193 name, j = self._scan_name(j, declstartpos)
194 if j < 0:
195 return j
196 c = rawdata[j:j+1]
197 if c == "":
198 return -1
199 if c == "(":
200 # an enumerated type; look for ')'
201 if ")" in rawdata[j:]:
202 j = rawdata.find(")", j) + 1
203 else:
204 return -1
205 while rawdata[j:j+1].isspace():
206 j = j + 1
207 if not rawdata[j:]:
208 # end of buffer, incomplete
209 return -1
210 else:
211 name, j = self._scan_name(j, declstartpos)
212 c = rawdata[j:j+1]
213 if not c:
214 return -1
215 if c in "'\"":
216 m = _declstringlit_match(rawdata, j)
217 if m:
218 j = m.end()
219 else:
220 return -1
221 c = rawdata[j:j+1]
222 if not c:
223 return -1
224 if c == "#":
225 if rawdata[j:] == "#":
226 # end of buffer
227 return -1
228 name, j = self._scan_name(j + 1, declstartpos)
229 if j < 0:
230 return j
231 c = rawdata[j:j+1]
232 if not c:
233 return -1
234 if c == '>':
235 # all done
236 return j + 1
238 # Internal -- scan past <!NOTATION declarations
239 def _parse_doctype_notation(self, i, declstartpos):
240 name, j = self._scan_name(i, declstartpos)
241 if j < 0:
242 return j
243 rawdata = self.rawdata
244 while 1:
245 c = rawdata[j:j+1]
246 if not c:
247 # end of buffer; incomplete
248 return -1
249 if c == '>':
250 return j + 1
251 if c in "'\"":
252 m = _declstringlit_match(rawdata, j)
253 if not m:
254 return -1
255 j = m.end()
256 else:
257 name, j = self._scan_name(j, declstartpos)
258 if j < 0:
259 return j
261 # Internal -- scan past <!ENTITY declarations
262 def _parse_doctype_entity(self, i, declstartpos):
263 rawdata = self.rawdata
264 if rawdata[i:i+1] == "%":
265 j = i + 1
266 while 1:
267 c = rawdata[j:j+1]
268 if not c:
269 return -1
270 if c in string.whitespace:
271 j = j + 1
272 else:
273 break
274 else:
275 j = i
276 name, j = self._scan_name(j, declstartpos)
277 if j < 0:
278 return j
279 while 1:
280 c = self.rawdata[j:j+1]
281 if not c:
282 return -1
283 if c in "'\"":
284 m = _declstringlit_match(rawdata, j)
285 if m:
286 j = m.end()
287 else:
288 return -1 # incomplete
289 elif c == ">":
290 return j + 1
291 else:
292 name, j = self._scan_name(j, declstartpos)
293 if j < 0:
294 return j
296 # Internal -- scan a name token and the new position and the token, or
297 # return -1 if we've reached the end of the buffer.
298 def _scan_name(self, i, declstartpos):
299 rawdata = self.rawdata
300 n = len(rawdata)
301 if i == n:
302 return None, -1
303 m = _declname_match(rawdata, i)
304 if m:
305 s = m.group()
306 name = s.strip()
307 if (i + len(s)) == n:
308 return None, -1 # end of buffer
309 return name.lower(), m.end()
310 else:
311 self.updatepos(declstartpos, i)
312 self.error("expected name token")