html_compose.translate_html

  1import inspect
  2import re
  3from functools import cache
  4from typing import Any
  5
  6from bs4 import BeautifulSoup, NavigableString, PageElement, Tag
  7from bs4.element import Doctype
  8
  9from . import BaseElement, escape_text
 10from . import elements as el_list
 11from .custom_element import CustomElement
 12from .util_funcs import safe_name
 13
 14SPEC_WS = r"[\t\n\r ]"
 15
 16
 17def _neighbor_tags(node: NavigableString) -> tuple[Tag | None, Tag | None]:
 18    """Return the closest Tag siblings to the left and right of a text node."""
 19
 20    def _find_tag_sibling(node, attr) -> Tag | None:
 21        sibling = getattr(node, attr)
 22        while sibling and not isinstance(sibling, Tag):
 23            sibling = getattr(sibling, attr)
 24        return sibling if isinstance(sibling, Tag) else None
 25
 26    return (
 27        _find_tag_sibling(node, "previous_sibling"),
 28        _find_tag_sibling(node, "next_sibling"),
 29    )
 30
 31
 32@cache
 33def get_phrasing_tags():
 34    """
 35    Get the list of phrasing tags from the HTML spec
 36    """
 37    result = []
 38    for e in dir(el_list):
 39        val = getattr(el_list, e)
 40        if isinstance(val, type) and issubclass(val, BaseElement):
 41            if val is BaseElement:
 42                continue
 43
 44            categories = getattr(val, "categories")
 45            tag = getattr(val, "tag")
 46            for c in categories:
 47                if "phrasing" in c:
 48                    result.append(tag)
 49    return result
 50
 51
 52def read_string(
 53    input_str: NavigableString,
 54    prev_tag: Tag | None,
 55    next_tag: Tag | None,
 56    phrasing_tags: list[str],
 57) -> str | None:
 58    """
 59    Helper to sort of 'auto-translate' HTML formatted strings into what
 60    they would be viewed as in a browser, which can then be represented in
 61    Python.
 62
 63    It collapses whitespace based on the context of the surrounding tags.
 64    """
 65    text = str(input_str)
 66
 67    # Trim if the previous or next tag is not an inline (phrasing) tag
 68    trim_left = False
 69    if prev_tag:
 70        trim_left = prev_tag.name not in phrasing_tags
 71    else:
 72        # No previous sibling, trim leading space
 73        trim_left = True
 74
 75    trim_right = False
 76    if next_tag:
 77        trim_right = next_tag.name not in phrasing_tags
 78    else:
 79        # No next sibling, trim trailing space
 80        trim_right = True
 81
 82    if trim_left:
 83        text = text.lstrip()
 84    if trim_right:
 85        text = text.rstrip()
 86
 87    # Collapse multiple whitespace characters into a single space
 88    result = re.sub(f"{SPEC_WS}+", " ", text)
 89
 90    if not result:
 91        return None
 92
 93    # If the original string was just whitespace and it got completely removed,
 94    # but it was between two inline tags, we should preserve a single space.
 95    if not result and input_str.strip() == "":
 96        if (
 97            prev_tag
 98            and next_tag
 99            and prev_tag.name in phrasing_tags
100            and next_tag.name in phrasing_tags
101        ):
102            return repr(" ")
103
104    if escape_text(result) != result:
105        # If the text (e.g., '&', '<'), would be escaped,
106        # To preserve the exact parsed string, we must wrap it in `unsafe_text`.
107        return f"unsafe_text({repr(result)})"
108
109    return repr(result)
110
111
112# HTML spec doesn't say this casually, but these are preformatted.
113WHITESPACE_PRE = ["pre", "textarea", "listing", "xmp"]
114
115
116def read_pre_string(input_str: NavigableString) -> str | None:
117    """
118    pre elements do the same as above, but remove the first newline
119    """
120    result = re.sub("^\n", "", input_str)
121    if not result:
122        return None
123
124    if escape_text(result) != result:
125        # If the text (e.g., '&', '<'), would be escaped,
126        # To preserve the exact parsed string, we must wrap it in `unsafe_text`.
127        return f"unsafe_text({repr(result)})"
128    return repr(result)
129
130
131class TranslateResult:
132    """
133    Class to hold the result of the translation
134    """
135
136    def __init__(
137        self,
138        elements: list[str],
139        tags: dict[str, Any],
140        import_statement: str = "",
141        custom_elements: list[str] | None = None,
142    ):
143        self.elements = elements
144        self.tags = tags
145        self.import_statement = import_statement
146        self.custom_elements = custom_elements or []
147
148    def as_array(self):
149        """
150        Return the elements as an array
151        """
152        sep = ",\n"
153        return f"[\n{sep.join(self.elements)}\n]"
154
155
156def is_preformatted(tag_name):
157    return tag_name in {"pre", "textarea"}
158
159
160def translate(
161    html: str, import_module: str | None = None, constructor: bool = False
162) -> TranslateResult:
163    """
164    Translate HTML string into Python code representing a similar HTML structure
165
166    We try to strip aesthetic line breaks from original HTML in this process.
167    """
168    soup = BeautifulSoup(html, features="html.parser")
169
170    tags: dict[str, Any] = {}
171    prefix = ""
172    if import_module is not None:
173        prefix = import_module + ("." if import_module else "")
174
175    custom_elements = set()
176
177    phrasing_tags = get_phrasing_tags()
178
179    import_unsafe_text = False
180
181    def process_element(element: PageElement) -> str | None:
182        if isinstance(element, Doctype):
183            dt: Doctype = element
184            tags["doctype"] = None
185            return f"doctype({repr(dt)})"
186        elif isinstance(element, NavigableString):
187            prev_tag, next_tag = _neighbor_tags(element)
188            return read_string(element, prev_tag, next_tag, phrasing_tags)
189
190        assert isinstance(element, Tag)
191        safe_tag_name = safe_name(element.name)
192        if safe_tag_name not in tags:
193            try:
194                tags[safe_tag_name] = getattr(el_list, safe_tag_name)
195            except AttributeError:
196                # This is a custom element, let's add it to the list
197                tags["CustomElement"] = None
198                tags[safe_tag_name] = CustomElement.create(safe_tag_name)
199                custom_elements.add(safe_tag_name)
200        is_custom = safe_tag_name in custom_elements
201        tag_cls = tags[safe_tag_name]
202
203        if is_custom:
204            # Custom elements aren't imported
205            result = [f"{safe_tag_name}"]
206        else:
207            result = [f"{prefix}{safe_tag_name}"]
208
209        if element.attrs:
210            param_attrs = {}
211            dict_attrs = {}
212            tag_keys = inspect.signature(tag_cls.__init__).parameters.keys()
213
214            for key, value in element.attrs.items():
215                # value bs4 gives us is sometimes
216                # like (key='rel', value=['preconnect'])
217                # If the attribute value is a list of one item, unwrap it
218                if isinstance(value, list) and len(value) == 1:
219                    value = value[0]
220
221                if key in ("attrs", "self", "children"):
222                    # These are params of the constructor but the HTML given
223                    # clashes with them
224                    dict_attrs[key] = value
225                    continue
226
227                safe_attr_name = safe_name(key)
228
229                if safe_attr_name in tag_keys:
230                    param_attrs[safe_attr_name] = value
231                else:
232                    # This is an unknown attribute,
233                    # let's include it as a dictionary key/value
234                    dict_attrs[key] = value
235
236            # Build element constructor call
237            result.append("(")
238
239            # Dict attributes first positionally
240            if dict_attrs:
241                result.append(repr(dict_attrs))
242
243            # Matching keyword args
244            if param_attrs:
245                if dict_attrs:
246                    result.append(", ")
247                params = []
248                for key, value in param_attrs.items():
249                    params.append(f"{key}={repr(value)}")
250                result.append(", ".join(params))
251
252            result.append(")")
253        else:
254            # If no attributes, we can skip the constructor call unless
255            if constructor:
256                result.append("()")
257
258        children: list[str] = []
259        child_nodes = list(element.children)
260        for i, child in enumerate(child_nodes):
261            if element.name in WHITESPACE_PRE and isinstance(
262                child, NavigableString
263            ):
264                processed = read_pre_string(child)
265                if processed:
266                    children.append(processed)
267                continue
268
269            if isinstance(child, NavigableString):
270                prev_tag, next_tag = _neighbor_tags(child)
271                processed = read_string(
272                    child, prev_tag, next_tag, phrasing_tags
273                )
274                if processed:
275                    children.append(processed)
276            elif isinstance(child, Tag):
277                processed = process_element(child)
278                if processed:
279                    children.append(processed)
280        for text_element in children:
281            if text_element.startswith("unsafe_text("):
282                nonlocal import_unsafe_text
283                import_unsafe_text = True
284        if children:
285            result.append("[")
286            result.append(", ".join(children))
287            result.append("]")
288        return "".join(result)
289
290    elements = [process_element(child) for child in soup.children]
291    import_statement = ""
292    if not import_module:
293        keys = [key for key in tags.keys() if key not in custom_elements]
294        if len(keys) > 3:
295            # Add parens
296            import_statement = f"from html_compose import ({', '.join(keys)})"
297        else:
298            import_statement = f"from html_compose import {', '.join(keys)}"
299
300        if import_unsafe_text:
301            import_statement += ", unsafe_text"
302    else:
303        if import_module == "html_compose":
304            import_statement = "import html_compose"
305        else:
306            import_statement = f"import html_compose as {import_module}"
307
308        if import_unsafe_text:
309            import_statement += "\nfrom html_compose import unsafe_text"
310
311    custom_el_stmts = [
312        f'{e} = {prefix}CustomElement.create("{e}")' for e in custom_elements
313    ]
314
315    return TranslateResult(
316        [e for e in elements if e], tags, import_statement, custom_el_stmts
317    )