diff options
Diffstat (limited to 'synapse/rest/media')
| -rw-r--r-- | synapse/rest/media/v1/preview_html.py | 63 |
1 files changed, 39 insertions, 24 deletions
diff --git a/synapse/rest/media/v1/preview_html.py b/synapse/rest/media/v1/preview_html.py index c826a13093..afe4e29758 100644 --- a/synapse/rest/media/v1/preview_html.py +++ b/synapse/rest/media/v1/preview_html.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. import codecs -import itertools import logging import re from typing import ( @@ -21,7 +20,7 @@ from typing import ( Dict, Generator, Iterable, - Optional, + List, Optional, Set, Union, ) @@ -354,7 +353,7 @@ def parse_html_description(tree: "etree.Element") -> Optional[str]: from lxml import etree - TAGS_TO_REMOVE = ( + TAGS_TO_REMOVE = { "header", "nav", "aside", @@ -369,31 +368,42 @@ def parse_html_description(tree: "etree.Element") -> Optional[str]: "img", "picture", etree.Comment, - ) + } # Split all the text nodes into paragraphs (by splitting on new # lines) text_nodes = ( re.sub(r"\s+", "\n", el).strip() - for el in _iterate_over_text(tree.find("body"), *TAGS_TO_REMOVE) + for el in _iterate_over_text(tree.find("body"), TAGS_TO_REMOVE) ) return summarize_paragraphs(text_nodes) def _iterate_over_text( - tree: "etree.Element", *tags_to_ignore: Union[str, "etree.Comment"] + tree: Optional["etree.Element"], + tags_to_ignore: Set[Union[str, "etree.Comment"]], + stack_limit: int = 1024, ) -> Generator[str, None, None]: """Iterate over the tree returning text nodes in a depth first fashion, skipping text nodes inside certain tags. + + Args: + tree: The parent element to iterate. Can be None if there isn't one. + tags_to_ignore: Set of tags to ignore + stack_limit: Maximum stack size limit for depth-first traversal. + Nodes will be dropped if this limit is hit, which may truncate the + textual result. + Intended to limit the maximum working memory when generating a preview. """ - # This is basically a stack that we extend using itertools.chain. - # This will either consist of an element to iterate over *or* a string + + if tree is None: + return + + # This is a stack whose items are elements to iterate over *or* strings # to be returned. - elements = iter([tree]) - while True: - el = next(elements, None) - if el is None: - return + elements: List[Union[str, "etree.Element"]] = [tree] + while elements: + el = elements.pop() if isinstance(el, str): yield el @@ -407,17 +417,22 @@ def _iterate_over_text( if el.text: yield el.text - # We add to the stack all the elements children, interspersed with - # each child's tail text (if it exists). The tail text of a node - # is text that comes *after* the node, so we always include it even - # if we ignore the child node. - elements = itertools.chain( - itertools.chain.from_iterable( # Basically a flatmap - [child, child.tail] if child.tail else [child] - for child in el.iterchildren() - ), - elements, - ) + # We add to the stack all the element's children, interspersed with + # each child's tail text (if it exists). + # + # We iterate in reverse order so that earlier pieces of text appear + # closer to the top of the stack. + for child in el.iterchildren(reversed=True): + if len(elements) > stack_limit: + # We've hit our limit for working memory + break + + if child.tail: + # The tail text of a node is text that comes *after* the node, + # so we always include it even if we ignore the child node. + elements.append(child.tail) + + elements.append(child) def summarize_paragraphs( |