# iterparse -- event-driven parsing
DEF __ITERPARSE_CHUNK_SIZE = 32768
cdef class iterparse:
u"""iterparse(self, source, events=("end",), tag=None, \
attribute_defaults=False, dtd_validation=False, \
load_dtd=False, no_network=True, remove_blank_text=False, \
remove_comments=False, remove_pis=False, encoding=None, \
html=False, recover=None, huge_tree=False, schema=None)
Incremental parser.
Parses XML into a tree and generates tuples (event, element) in a
SAX-like fashion. ``event`` is any of 'start', 'end', 'start-ns',
'end-ns'.
For 'start' and 'end', ``element`` is the Element that the parser just
found opening or closing. For 'start-ns', it is a tuple (prefix, URI) of
a new namespace declaration. For 'end-ns', it is simply None. Note that
all start and end events are guaranteed to be properly nested.
The keyword argument ``events`` specifies a sequence of event type names
that should be generated. By default, only 'end' events will be
generated.
The additional ``tag`` argument restricts the 'start' and 'end' events to
those elements that match the given tag. The ``tag`` argument can also be
a sequence of tags to allow matching more than one tag. By default,
events are generated for all elements. Note that the 'start-ns' and
'end-ns' events are not impacted by this restriction.
The other keyword arguments in the constructor are mainly based on the
libxml2 parser configuration. A DTD will also be loaded if validation or
attribute default values are requested.
Available boolean keyword arguments:
- attribute_defaults: read default attributes from DTD
- dtd_validation: validate (if DTD is available)
- load_dtd: use DTD for parsing
- no_network: prevent network access for related files
- remove_blank_text: discard blank text nodes
- remove_comments: discard comments
- remove_pis: discard processing instructions
- strip_cdata: replace CDATA sections by normal text content (default: True)
- compact: safe memory for short text content (default: True)
- resolve_entities: replace entities by their text value (default: True)
- huge_tree: disable security restrictions and support very deep trees
and very long text content (only affects libxml2 2.7+)
- html: parse input as HTML (default: XML)
- recover: try hard to parse through broken input (default: True for HTML,
False otherwise)
Other keyword arguments:
- encoding: override the document encoding
- schema: an XMLSchema to validate against
"""
cdef _FeedParser _parser
cdef object _tag
cdef object _events
cdef readonly object root
cdef object _source
cdef object _filename
cdef object _error
cdef bint _close_source_after_read
def __init__(self, source, events=(u"end",), *, tag=None,
attribute_defaults=False, dtd_validation=False,
load_dtd=False, no_network=True, remove_blank_text=False,
compact=True, resolve_entities=True, remove_comments=False,
remove_pis=False, strip_cdata=True, encoding=None,
html=False, recover=None, huge_tree=False, collect_ids=True,
XMLSchema schema=None):
if not hasattr(source, 'read'):
source = _getFSPathOrObject(source)
self._filename = source
if python.IS_PYTHON2:
source = _encodeFilename(source)
source = open(source, 'rb')
self._close_source_after_read = True
else:
self._filename = _getFilenameForFile(source)
self._close_source_after_read = False
if recover is None:
recover = html
if html:
# make sure we're not looking for namespaces
events = [event for event in events
if event not in ('start-ns', 'end-ns')]
parser = HTMLPullParser(
events,
tag=tag,
recover=recover,
base_url=self._filename,
encoding=encoding,
remove_blank_text=remove_blank_text,
remove_comments=remove_comments,
remove_pis=remove_pis,
strip_cdata=strip_cdata,
no_network=no_network,
target=None, # TODO
schema=schema,
compact=compact)
else:
parser = XMLPullParser(
events,
tag=tag,
recover=recover,
base_url=self._filename,
encoding=encoding,
attribute_defaults=attribute_defaults,
dtd_validation=dtd_validation,
load_dtd=load_dtd,
no_network=no_network,
schema=schema,
huge_tree=huge_tree,
remove_blank_text=remove_blank_text,
resolve_entities=resolve_entities,
remove_comments=remove_comments,
remove_pis=remove_pis,
strip_cdata=strip_cdata,
collect_ids=True,
target=None, # TODO
compact=compact)
self._events = parser.read_events()
self._parser = parser
self._source = source
@property
def error_log(self):
"""The error log of the last (or current) parser run.
"""
return self._parser.feed_error_log
@property
def resolvers(self):
"""The custom resolver registry of the last (or current) parser run.
"""
return self._parser.resolvers
@property
def version(self):
"""The version of the underlying XML parser."""
return self._parser.version
def set_element_class_lookup(self, ElementClassLookup lookup = None):
u"""set_element_class_lookup(self, lookup = None)
Set a lookup scheme for element classes generated from this parser.
Reset it by passing None or nothing.
"""
self._parser.set_element_class_lookup(lookup)
def makeelement(self, _tag, attrib=None, nsmap=None, **_extra):
u"""makeelement(self, _tag, attrib=None, nsmap=None, **_extra)
Creates a new element associated with this parser.
"""
self._parser.makeelement(
_tag, attrib=None, nsmap=None, **_extra)
@cython.final
cdef _close_source(self):
if self._source is None:
return
if not self._close_source_after_read:
self._source = None
return
try:
close = self._source.close
except AttributeError:
close = None
finally:
self._source = None
if close is not None:
close()
def __iter__(self):
return self
def __next__(self):
try:
return next(self._events)
except StopIteration:
pass
context = <_SaxParserContext>self._parser._getPushParserContext()
if self._source is not None:
done = False
while not done:
try:
done = self._read_more_events(context)
return next(self._events)
except StopIteration:
pass # no events yet
except Exception as e:
self._error = e
self._close_source()
try:
return next(self._events)
except StopIteration:
break
# nothing left to read or return
if self._error is not None:
error = self._error
self._error = None
raise error
if (context._validator is not None
and not context._validator.isvalid()):
_raiseParseError(context._c_ctxt, self._filename,
context._error_log)
# no errors => all done
raise StopIteration
@cython.final
cdef bint _read_more_events(self, _SaxParserContext context) except -123:
data = self._source.read(__ITERPARSE_CHUNK_SIZE)
if not isinstance(data, bytes):
self._close_source()
raise TypeError("reading file objects must return bytes objects")
if not data:
try:
self.root = self._parser.close()
finally:
self._close_source()
return True
self._parser.feed(data)
return False
cdef enum _IterwalkSkipStates:
IWSKIP_NEXT_IS_START
IWSKIP_SKIP_NEXT
IWSKIP_CAN_SKIP
IWSKIP_CANNOT_SKIP
cdef class iterwalk:
u"""iterwalk(self, element_or_tree, events=("end",), tag=None)
A tree walker that generates events from an existing tree as if it
was parsing XML data with ``iterparse()``.
Just as for ``iterparse()``, the ``tag`` argument can be a single tag or a
sequence of tags.
After receiving a 'start' or 'start-ns' event, the children and
descendants of the current element can be excluded from iteration
by calling the ``skip_subtree()`` method.
"""
cdef _MultiTagMatcher _matcher
cdef list _node_stack
cdef list _events
cdef object _pop_event
cdef object _include_siblings
cdef int _index
cdef int _event_filter
cdef _IterwalkSkipStates _skip_state
def __init__(self, element_or_tree, events=(u"end",), tag=None):
cdef _Element root
cdef int ns_count
root = _rootNodeOrRaise(element_or_tree)
self._event_filter = _buildParseEventFilter(events)
if tag is None or tag == '*':
self._matcher = None
else:
self._matcher = _MultiTagMatcher.__new__(_MultiTagMatcher, tag)
self._node_stack = []
self._events = []
self._pop_event = self._events.pop
self._skip_state = IWSKIP_CANNOT_SKIP # ignore all skip requests by default
if self._event_filter:
self._index = 0
if self._matcher is not None and self._event_filter & PARSE_EVENT_FILTER_START:
self._matcher.cacheTags(root._doc)
# When processing an ElementTree, add events for the preceding comments/PIs.
if self._event_filter & (PARSE_EVENT_FILTER_COMMENT | PARSE_EVENT_FILTER_PI):
if isinstance(element_or_tree, _ElementTree):
self._include_siblings = root
for elem in list(root.itersiblings(preceding=True))[::-1]:
if self._event_filter & PARSE_EVENT_FILTER_COMMENT and elem.tag is Comment:
self._events.append((u'comment', elem))
elif self._event_filter & PARSE_EVENT_FILTER_PI and elem.tag is PI:
self._events.append((u'pi', elem))
ns_count = self._start_node(root)
self._node_stack.append( (root, ns_count) )
else:
self._index = -1
def __iter__(self):
return self
def __next__(self):
cdef xmlNode* c_child
cdef _Element node
cdef _Element next_node
cdef int ns_count = 0
if self._events:
return self._next_event()
if self._matcher is not None and self._index >= 0:
node = self._node_stack[self._index][0]
self._matcher.cacheTags(node._doc)
# find next node
while self._index >= 0:
node = self._node_stack[self._index][0]
if self._skip_state == IWSKIP_SKIP_NEXT:
c_child = NULL
else:
c_child = self._process_non_elements(
node._doc, _findChildForwards(node._c_node, 0))
self._skip_state = IWSKIP_CANNOT_SKIP
while c_child is NULL:
# back off through parents
self._index -= 1
node = self._end_node()
if self._index < 0:
break
c_child = self._process_non_elements(
node._doc, _nextElement(node._c_node))
if c_child is not NULL:
next_node = _elementFactory(node._doc, c_child)
if self._event_filter & (PARSE_EVENT_FILTER_START |
PARSE_EVENT_FILTER_START_NS):
ns_count = self._start_node(next_node)
elif self._event_filter & PARSE_EVENT_FILTER_END_NS:
ns_count = _countNsDefs(next_node._c_node)
self._node_stack.append( (next_node, ns_count) )
self._index += 1
if self._events:
return self._next_event()
if self._include_siblings is not None:
node, self._include_siblings = self._include_siblings, None
self._process_non_elements(node._doc, _nextElement(node._c_node))
if self._events:
return self._next_event()
raise StopIteration
@cython.final
cdef xmlNode* _process_non_elements(self, _Document doc, xmlNode* c_node):
while c_node is not NULL and c_node.type != tree.XML_ELEMENT_NODE:
if c_node.type == tree.XML_COMMENT_NODE:
if self._event_filter & PARSE_EVENT_FILTER_COMMENT:
self._events.append(
(u"comment", _elementFactory(doc, c_node)))
c_node = _nextElement(c_node)
elif c_node.type == tree.XML_PI_NODE:
if self._event_filter & PARSE_EVENT_FILTER_PI:
self._events.append(
(u"pi", _elementFactory(doc, c_node)))
c_node = _nextElement(c_node)
else:
break
return c_node
@cython.final
cdef _next_event(self):
if self._skip_state == IWSKIP_NEXT_IS_START:
if self._events[0][0] in (u'start', u'start-ns'):
self._skip_state = IWSKIP_CAN_SKIP
return self._pop_event(0)
def skip_subtree(self):
"""Prevent descending into the current subtree.
Instead, the next returned event will be the 'end' event of the current element
(if included), ignoring any children or descendants.
This has no effect right after an 'end' or 'end-ns' event.
"""
if self._skip_state == IWSKIP_CAN_SKIP:
self._skip_state = IWSKIP_SKIP_NEXT
@cython.final
cdef int _start_node(self, _Element node) except -1:
cdef int ns_count
if self._event_filter & PARSE_EVENT_FILTER_START_NS:
ns_count = _appendStartNsEvents(node._c_node, self._events)
if self._events:
self._skip_state = IWSKIP_NEXT_IS_START
elif self._event_filter & PARSE_EVENT_FILTER_END_NS:
ns_count = _countNsDefs(node._c_node)
else:
ns_count = 0
if self._event_filter & PARSE_EVENT_FILTER_START:
if self._matcher is None or self._matcher.matches(node._c_node):
self._events.append( (u"start", node) )
self._skip_state = IWSKIP_NEXT_IS_START
return ns_count
@cython.final
cdef _Element _end_node(self):
cdef _Element node
cdef int i, ns_count
node, ns_count = self._node_stack.pop()
if self._event_filter & PARSE_EVENT_FILTER_END:
if self._matcher is None or self._matcher.matches(node._c_node):
self._events.append( (u"end", node) )
if self._event_filter & PARSE_EVENT_FILTER_END_NS and ns_count:
event = (u"end-ns", None)
for i in range(ns_count):
self._events.append(event)
return node
cdef int _countNsDefs(xmlNode* c_node):
cdef xmlNs* c_ns
cdef int count
count = 0
c_ns = c_node.nsDef
while c_ns is not NULL:
count += (c_ns.href is not NULL)
c_ns = c_ns.next
return count
cdef int _appendStartNsEvents(xmlNode* c_node, list event_list) except -1:
cdef xmlNs* c_ns
cdef int count
count = 0
c_ns = c_node.nsDef
while c_ns is not NULL:
if c_ns.href:
ns_tuple = (funicodeOrEmpty(c_ns.prefix),
funicode(c_ns.href))
event_list.append( (u"start-ns", ns_tuple) )
count += 1
c_ns = c_ns.next
return count