diff --git a/pyWebLayout/io/readers/html_extraction.py b/pyWebLayout/io/readers/html_extraction.py index 8998066..b962c4f 100644 --- a/pyWebLayout/io/readers/html_extraction.py +++ b/pyWebLayout/io/readers/html_extraction.py @@ -8,6 +8,7 @@ Each handler function has a robust signature that handles style hints, CSS class from typing import List, Dict, Any, Optional, Union, Callable, Tuple, NamedTuple from bs4 import BeautifulSoup, Tag, NavigableString +from bs4.element import CData, Comment, Doctype, ProcessingInstruction from pyWebLayout.abstract.inline import Word from pyWebLayout.abstract.block import ( Block, @@ -369,6 +370,24 @@ def extract_text_content(element: Tag, context: StyleContext) -> List[Word]: element: BeautifulSoup Tag object context: Current style context + Returns: + List of Word objects (including LinkedWord for hyperlinks) + """ + return extract_words_from_nodes(list(element.children), context) + + +def extract_words_from_nodes(nodes: List, context: StyleContext) -> List[Word]: + """ + Extract words from a sequence of sibling nodes. + + Separated from extract_text_content so that a container holding a mix of + inline and block children can hand over just the inline runs, without + building a synthetic element to wrap them in. + + Args: + nodes: BeautifulSoup nodes (Tags and NavigableStrings) in document order + context: Current style context + Returns: List of Word objects (including LinkedWord for hyperlinks) """ @@ -377,7 +396,12 @@ def extract_text_content(element: Tag, context: StyleContext) -> List[Word]: words = [] - for child in element.children: + for child in nodes: + # Comments and processing instructions are NavigableString subclasses; + # their text is markup, not content. + if isinstance(child, (Comment, Doctype, CData, ProcessingInstruction)): + continue + if isinstance(child, NavigableString): # Plain text - split into words text = str(child).strip() @@ -466,6 +490,93 @@ def extract_text_content(element: Tag, context: StyleContext) -> List[Word]: return words +# Tags that flow within a line of text rather than forming a block of their own. +# They carry no handler of their own: extract_words_from_nodes consumes them, +# applying their styling to the words they contain. +INLINE_TAGS = frozenset({ + "a", "span", "strong", "b", "em", "i", "u", "s", "del", "ins", "mark", + "small", "sub", "sup", "code", "q", "cite", "abbr", "time", +}) + + +def is_inline(node) -> bool: + """ + Whether a node belongs to a run of text rather than standing as its own block. + + Args: + node: A BeautifulSoup Tag or NavigableString + + Returns: + True for text and inline tags, False for block-level tags + """ + if isinstance(node, Tag): + return node.name.lower() in INLINE_TAGS + if isinstance(node, (Comment, Doctype, CData, ProcessingInstruction)): + return False + return isinstance(node, NavigableString) + + +def process_block_children(element: Tag, context: StyleContext) -> List[Block]: + """ + Process a container's children into a list of blocks. + + Containers may hold a mix of inline and block content. Consecutive inline + children are gathered into a run and become one Paragraph; a block child ends + the current run and is processed by its own handler. This is the single entry + point for every container that is not itself a paragraph - div, li, td, th, + blockquote and the semantic containers. + + Without this, inline tags reach process_element, whose handler for them is + ignore_handler, and their text is silently dropped. + + Args: + element: The container element + context: Current style context + + Returns: + Blocks in document order + """ + blocks: List[Block] = [] + run: List = [] + + def flush_run(): + """Turn the pending inline run into a paragraph, if it holds any words.""" + if not run: + return + words = extract_words_from_nodes(run, context) + run.clear() + if words: + paragraph = Paragraph(context.font) + for word in words: + paragraph.add_word(word) + blocks.append(paragraph) + + for child in element.children: + #
ends the current line of text and starts a new one. + if isinstance(child, Tag) and child.name.lower() == "br": + flush_run() + continue + + if is_inline(child): + run.append(child) + continue + + if not isinstance(child, Tag): + continue # comments and similar + + flush_run() + child_context = apply_element_styling(context, child) + result = process_element(child, child_context) + if result: + if isinstance(result, list): + blocks.extend(result) + else: + blocks.append(result) + + flush_run() + return blocks + + def process_element( element: Tag, context: StyleContext ) -> Union[Block, List[Block], None]: @@ -557,17 +668,7 @@ def paragraph_handler(element: Tag, context: StyleContext) -> Union[Paragraph, L def div_handler(element: Tag, context: StyleContext) -> List[Block]: """Handle
elements - treat as generic container.""" - blocks = [] - for child in element.children: - if isinstance(child, Tag): - child_context = apply_element_styling(context, child) - result = process_element(child, child_context) - if result: - if isinstance(result, list): - blocks.extend(result) - else: - blocks.append(result) - return blocks + return process_block_children(element, context) def heading_handler(element: Tag, context: StyleContext) -> Heading: @@ -592,16 +693,8 @@ def heading_handler(element: Tag, context: StyleContext) -> Heading: def blockquote_handler(element: Tag, context: StyleContext) -> Quote: """Handle
elements.""" quote = Quote(context.font) - for child in element.children: - if isinstance(child, Tag): - child_context = apply_element_styling(context, child) - result = process_element(child, child_context) - if result: - if isinstance(result, list): - for block in result: - quote.add_block(block) - else: - quote.add_block(result) + for block in process_block_children(element, context): + quote.add_block(block) return quote @@ -655,28 +748,8 @@ def ordered_list_handler(element: Tag, context: StyleContext) -> HList: def list_item_handler(element: Tag, context: StyleContext) -> ListItem: """Handle
  • elements.""" list_item = ListItem(None, context.font) - - for child in element.children: - if isinstance(child, Tag): - child_context = apply_element_styling(context, child) - result = process_element(child, child_context) - if result: - if isinstance(result, list): - for block in result: - list_item.add_block(block) - else: - list_item.add_block(result) - elif isinstance(child, NavigableString): - # Direct text in list item - create paragraph - text = str(child).strip() - if text: - paragraph = Paragraph(context.font) - words = text.split() - for word_text in words: - if word_text: - paragraph.add_word(Word(word_text, context.font)) - list_item.add_block(paragraph) - + for block in process_block_children(element, context): + list_item.add_block(block) return list_item @@ -728,27 +801,8 @@ def table_cell_handler(element: Tag, context: StyleContext) -> TableCell: rowspan = int(context.element_attributes.get("rowspan", 1)) cell = TableCell(False, colspan, rowspan, context.font) - # Process cell content - for child in element.children: - if isinstance(child, Tag): - child_context = apply_element_styling(context, child) - result = process_element(child, child_context) - if result: - if isinstance(result, list): - for block in result: - cell.add_block(block) - else: - cell.add_block(result) - elif isinstance(child, NavigableString): - # Direct text in cell - create paragraph - text = str(child).strip() - if text: - paragraph = Paragraph(context.font) - words = text.split() - for word_text in words: - if word_text: - paragraph.add_word(Word(word_text, context.font)) - cell.add_block(paragraph) + for block in process_block_children(element, context): + cell.add_block(block) return cell @@ -759,26 +813,8 @@ def table_header_cell_handler(element: Tag, context: StyleContext) -> TableCell: rowspan = int(context.element_attributes.get("rowspan", 1)) cell = TableCell(True, colspan, rowspan, context.font) - # Process cell content (same as td) - for child in element.children: - if isinstance(child, Tag): - child_context = apply_element_styling(context, child) - result = process_element(child, child_context) - if result: - if isinstance(result, list): - for block in result: - cell.add_block(block) - else: - cell.add_block(result) - elif isinstance(child, NavigableString): - text = str(child).strip() - if text: - paragraph = Paragraph(context.font) - words = text.split() - for word_text in words: - if word_text: - paragraph.add_word(Word(word_text, context.font)) - cell.add_block(paragraph) + for block in process_block_children(element, context): + cell.add_block(block) return cell diff --git a/tests/io_tests/test_inline_content_in_containers.py b/tests/io_tests/test_inline_content_in_containers.py new file mode 100644 index 0000000..7c7e4c1 --- /dev/null +++ b/tests/io_tests/test_inline_content_in_containers.py @@ -0,0 +1,169 @@ +""" +Regression tests for inline content inside block containers (spec S1). + +Inline tags are registered to ignore_handler because they are meant to be +consumed by extract_text_content. Only

    and

    -

    ever called it, so +every other container - div, li, td, th, blockquote - iterated its children as +blocks, and inline tags returned None. Their text was silently discarded, and +bare text nodes each became a separate paragraph. +""" + +import pytest + +from pyWebLayout.abstract.block import ( + HList, + Paragraph, + Quote, + Table, +) +from pyWebLayout.abstract.inline import LinkedWord, Word +from pyWebLayout.io.readers.html_extraction import parse_html_string + + +def words_of(block): + return [w.text for w in getattr(block, 'words', [])] + + +def all_words(blocks): + out = [] + for block in blocks: + out.extend(words_of(block)) + return out + + +def cell_blocks(table): + for _, row in table.all_rows(): + for cell in row.cells(): + yield list(cell.blocks()) + + +EXPECTED = ["hello", "world", "again"] + + +class TestInlineContentIsKept: + """The same markup must survive in every container.""" + + def test_paragraph_control(self): + """

    already worked - this is the reference behaviour.""" + blocks = parse_html_string("

    hello world again

    ") + assert all_words(blocks) == EXPECTED + + def test_div(self): + blocks = parse_html_string("
    hello world again
    ") + assert all_words(blocks) == EXPECTED + + def test_list_item(self): + blocks = parse_html_string("
    • hello world again
    ") + hlist = next(b for b in blocks if isinstance(b, HList)) + item = list(hlist.items())[0] + assert all_words(item.blocks()) == EXPECTED + + def test_table_cell(self): + blocks = parse_html_string( + "
    hello world again
    ") + table = next(b for b in blocks if isinstance(b, Table)) + assert all_words(next(cell_blocks(table))) == EXPECTED + + def test_table_header_cell(self): + blocks = parse_html_string( + "
    hello world again
    ") + table = next(b for b in blocks if isinstance(b, Table)) + assert all_words(next(cell_blocks(table))) == EXPECTED + + def test_blockquote(self): + blocks = parse_html_string("
    hello world again
    ") + quote = next(b for b in blocks if isinstance(b, Quote)) + assert all_words(quote.blocks()) == EXPECTED + + +class TestInlineRunsCoalesce: + """A run of inline content is one paragraph, not one per text node.""" + + def test_div_yields_a_single_paragraph(self): + blocks = parse_html_string("
    a b c
    ") + paragraphs = [b for b in blocks if isinstance(b, Paragraph)] + assert len(paragraphs) == 1, f"expected one paragraph, got {len(blocks)} blocks" + assert words_of(paragraphs[0]) == ["a", "b", "c"] + + def test_cell_yields_a_single_paragraph(self): + blocks = parse_html_string("
    a b c
    ") + table = next(b for b in blocks if isinstance(b, Table)) + cell = next(cell_blocks(table)) + assert len(cell) == 1 + assert words_of(cell[0]) == ["a", "b", "c"] + + def test_block_child_splits_the_run(self): + """Inline runs either side of a block child stay separate, in order.""" + blocks = parse_html_string( + "
    before

    middle

    after
    ") + table = next(b for b in blocks if isinstance(b, Table)) + cell = next(cell_blocks(table)) + assert [words_of(b) for b in cell] == [["before"], ["middle"], ["after"]] + + def test_line_break_splits_the_run(self): + blocks = parse_html_string("
    first
    second
    ") + paragraphs = [b for b in blocks if isinstance(b, Paragraph)] + assert [words_of(p) for p in paragraphs] == [["first"], ["second"]] + + def test_whitespace_between_blocks_makes_no_paragraph(self): + blocks = parse_html_string("
    \n

    one

    \n

    two

    \n
    ") + assert [words_of(b) for b in blocks] == [["one"], ["two"]] + + +class TestLinksSurvive: + """ must produce LinkedWord wherever it appears.""" + + def test_link_in_cell(self): + blocks = parse_html_string( + '
    link text
    ') + table = next(b for b in blocks if isinstance(b, Table)) + cell = next(cell_blocks(table)) + found = [w for b in cell for w in getattr(b, 'words', [])] + + assert [w.text for w in found] == ["link", "text"] + linked = [w for w in found if isinstance(w, LinkedWord)] + assert len(linked) == 1 + assert linked[0].location == "http://x" + + def test_link_in_div(self): + blocks = parse_html_string('
    see Section 2 now
    ') + found = [w for b in blocks for w in getattr(b, 'words', [])] + assert [w.text for w in found] == ["see", "Section", "2", "now"] + assert all(isinstance(w, LinkedWord) for w in found[1:3]) + + def test_link_in_list_item(self): + blocks = parse_html_string('') + hlist = next(b for b in blocks if isinstance(b, HList)) + item = hlist._items[0] + found = [w for b in item.blocks() for w in getattr(b, 'words', [])] + assert [w.text for w in found] == ["click", "here"] + assert isinstance(found[0], LinkedWord) + + +class TestNestedContainers: + + def test_div_in_div(self): + blocks = parse_html_string("
    outer
    inner
    tail
    ") + assert [words_of(b) for b in blocks] == [["outer"], ["inner"], ["tail"]] + + def test_block_children_still_pass_through(self): + blocks = parse_html_string("

    Title

    Body

    ") + assert len(blocks) == 2 + assert words_of(blocks[0]) == ["Title"] + assert words_of(blocks[1]) == ["Body"] + + def test_cell_containing_a_list(self): + blocks = parse_html_string( + "
    intro
    • item
    ") + table = next(b for b in blocks if isinstance(b, Table)) + cell = next(cell_blocks(table)) + assert isinstance(cell[0], Paragraph) + assert words_of(cell[0]) == ["intro"] + assert isinstance(cell[1], HList) + + +class TestComments: + + def test_comment_text_is_not_content(self): + blocks = parse_html_string("
    realtext
    ") + assert all_words(blocks) == ["real", "text"]