diff --git a/pyWebLayout/io/readers/html_extraction.py b/pyWebLayout/io/readers/html_extraction.py
index 8998066..b962c4f 100644
--- a/pyWebLayout/io/readers/html_extraction.py
+++ b/pyWebLayout/io/readers/html_extraction.py
@@ -8,6 +8,7 @@ Each handler function has a robust signature that handles style hints, CSS class
from typing import List, Dict, Any, Optional, Union, Callable, Tuple, NamedTuple
from bs4 import BeautifulSoup, Tag, NavigableString
+from bs4.element import CData, Comment, Doctype, ProcessingInstruction
from pyWebLayout.abstract.inline import Word
from pyWebLayout.abstract.block import (
Block,
@@ -369,6 +370,24 @@ def extract_text_content(element: Tag, context: StyleContext) -> List[Word]:
element: BeautifulSoup Tag object
context: Current style context
+ Returns:
+ List of Word objects (including LinkedWord for hyperlinks)
+ """
+ return extract_words_from_nodes(list(element.children), context)
+
+
+def extract_words_from_nodes(nodes: List, context: StyleContext) -> List[Word]:
+ """
+ Extract words from a sequence of sibling nodes.
+
+ Separated from extract_text_content so that a container holding a mix of
+ inline and block children can hand over just the inline runs, without
+ building a synthetic element to wrap them in.
+
+ Args:
+ nodes: BeautifulSoup nodes (Tags and NavigableStrings) in document order
+ context: Current style context
+
Returns:
List of Word objects (including LinkedWord for hyperlinks)
"""
@@ -377,7 +396,12 @@ def extract_text_content(element: Tag, context: StyleContext) -> List[Word]:
words = []
- for child in element.children:
+ for child in nodes:
+ # Comments and processing instructions are NavigableString subclasses;
+ # their text is markup, not content.
+ if isinstance(child, (Comment, Doctype, CData, ProcessingInstruction)):
+ continue
+
if isinstance(child, NavigableString):
# Plain text - split into words
text = str(child).strip()
@@ -466,6 +490,93 @@ def extract_text_content(element: Tag, context: StyleContext) -> List[Word]:
return words
+# Tags that flow within a line of text rather than forming a block of their own.
+# They carry no handler of their own: extract_words_from_nodes consumes them,
+# applying their styling to the words they contain.
+INLINE_TAGS = frozenset({
+ "a", "span", "strong", "b", "em", "i", "u", "s", "del", "ins", "mark",
+ "small", "sub", "sup", "code", "q", "cite", "abbr", "time",
+})
+
+
+def is_inline(node) -> bool:
+ """
+ Whether a node belongs to a run of text rather than standing as its own block.
+
+ Args:
+ node: A BeautifulSoup Tag or NavigableString
+
+ Returns:
+ True for text and inline tags, False for block-level tags
+ """
+ if isinstance(node, Tag):
+ return node.name.lower() in INLINE_TAGS
+ if isinstance(node, (Comment, Doctype, CData, ProcessingInstruction)):
+ return False
+ return isinstance(node, NavigableString)
+
+
+def process_block_children(element: Tag, context: StyleContext) -> List[Block]:
+ """
+ Process a container's children into a list of blocks.
+
+ Containers may hold a mix of inline and block content. Consecutive inline
+ children are gathered into a run and become one Paragraph; a block child ends
+ the current run and is processed by its own handler. This is the single entry
+ point for every container that is not itself a paragraph - div, li, td, th,
+ blockquote and the semantic containers.
+
+ Without this, inline tags reach process_element, whose handler for them is
+ ignore_handler, and their text is silently dropped.
+
+ Args:
+ element: The container element
+ context: Current style context
+
+ Returns:
+ Blocks in document order
+ """
+ blocks: List[Block] = []
+ run: List = []
+
+ def flush_run():
+ """Turn the pending inline run into a paragraph, if it holds any words."""
+ if not run:
+ return
+ words = extract_words_from_nodes(run, context)
+ run.clear()
+ if words:
+ paragraph = Paragraph(context.font)
+ for word in words:
+ paragraph.add_word(word)
+ blocks.append(paragraph)
+
+ for child in element.children:
+ #
ends the current line of text and starts a new one.
+ if isinstance(child, Tag) and child.name.lower() == "br":
+ flush_run()
+ continue
+
+ if is_inline(child):
+ run.append(child)
+ continue
+
+ if not isinstance(child, Tag):
+ continue # comments and similar
+
+ flush_run()
+ child_context = apply_element_styling(context, child)
+ result = process_element(child, child_context)
+ if result:
+ if isinstance(result, list):
+ blocks.extend(result)
+ else:
+ blocks.append(result)
+
+ flush_run()
+ return blocks
+
+
def process_element(
element: Tag, context: StyleContext
) -> Union[Block, List[Block], None]:
@@ -557,17 +668,7 @@ def paragraph_handler(element: Tag, context: StyleContext) -> Union[Paragraph, L
def div_handler(element: Tag, context: StyleContext) -> List[Block]:
"""Handle
elements.""" quote = Quote(context.font) - for child in element.children: - if isinstance(child, Tag): - child_context = apply_element_styling(context, child) - result = process_element(child, child_context) - if result: - if isinstance(result, list): - for block in result: - quote.add_block(block) - else: - quote.add_block(result) + for block in process_block_children(element, context): + quote.add_block(block) return quote @@ -655,28 +748,8 @@ def ordered_list_handler(element: Tag, context: StyleContext) -> HList: def list_item_handler(element: Tag, context: StyleContext) -> ListItem: """Handleelements.""" list_item = ListItem(None, context.font) - - for child in element.children: - if isinstance(child, Tag): - child_context = apply_element_styling(context, child) - result = process_element(child, child_context) - if result: - if isinstance(result, list): - for block in result: - list_item.add_block(block) - else: - list_item.add_block(result) - elif isinstance(child, NavigableString): - # Direct text in list item - create paragraph - text = str(child).strip() - if text: - paragraph = Paragraph(context.font) - words = text.split() - for word_text in words: - if word_text: - paragraph.add_word(Word(word_text, context.font)) - list_item.add_block(paragraph) - + for block in process_block_children(element, context): + list_item.add_block(block) return list_item @@ -728,27 +801,8 @@ def table_cell_handler(element: Tag, context: StyleContext) -> TableCell: rowspan = int(context.element_attributes.get("rowspan", 1)) cell = TableCell(False, colspan, rowspan, context.font) - # Process cell content - for child in element.children: - if isinstance(child, Tag): - child_context = apply_element_styling(context, child) - result = process_element(child, child_context) - if result: - if isinstance(result, list): - for block in result: - cell.add_block(block) - else: - cell.add_block(result) - elif isinstance(child, NavigableString): - # Direct text in cell - create paragraph - text = str(child).strip() - if text: - paragraph = Paragraph(context.font) - words = text.split() - for word_text in words: - if word_text: - paragraph.add_word(Word(word_text, context.font)) - cell.add_block(paragraph) + for block in process_block_children(element, context): + cell.add_block(block) return cell @@ -759,26 +813,8 @@ def table_header_cell_handler(element: Tag, context: StyleContext) -> TableCell: rowspan = int(context.element_attributes.get("rowspan", 1)) cell = TableCell(True, colspan, rowspan, context.font) - # Process cell content (same as td) - for child in element.children: - if isinstance(child, Tag): - child_context = apply_element_styling(context, child) - result = process_element(child, child_context) - if result: - if isinstance(result, list): - for block in result: - cell.add_block(block) - else: - cell.add_block(result) - elif isinstance(child, NavigableString): - text = str(child).strip() - if text: - paragraph = Paragraph(context.font) - words = text.split() - for word_text in words: - if word_text: - paragraph.add_word(Word(word_text, context.font)) - cell.add_block(paragraph) + for block in process_block_children(element, context): + cell.add_block(block) return cell diff --git a/tests/io_tests/test_inline_content_in_containers.py b/tests/io_tests/test_inline_content_in_containers.py new file mode 100644 index 0000000..7c7e4c1 --- /dev/null +++ b/tests/io_tests/test_inline_content_in_containers.py @@ -0,0 +1,169 @@ +""" +Regression tests for inline content inside block containers (spec S1). + +Inline tags are registered to ignore_handler because they are meant to be +consumed by extract_text_content. Only and
-
ever called it, so +every other container - div, li, td, th, blockquote - iterated its children as +blocks, and inline tags returned None. Their text was silently discarded, and +bare text nodes each became a separate paragraph. +""" + +import pytest + +from pyWebLayout.abstract.block import ( + HList, + Paragraph, + Quote, + Table, +) +from pyWebLayout.abstract.inline import LinkedWord, Word +from pyWebLayout.io.readers.html_extraction import parse_html_string + + +def words_of(block): + return [w.text for w in getattr(block, 'words', [])] + + +def all_words(blocks): + out = [] + for block in blocks: + out.extend(words_of(block)) + return out + + +def cell_blocks(table): + for _, row in table.all_rows(): + for cell in row.cells(): + yield list(cell.blocks()) + + +EXPECTED = ["hello", "world", "again"] + + +class TestInlineContentIsKept: + """The same markup must survive in every container.""" + + def test_paragraph_control(self): + """
already worked - this is the reference behaviour.""" + blocks = parse_html_string("
hello world again
") + assert all_words(blocks) == EXPECTED + + def test_div(self): + blocks = parse_html_string("hello world again") + assert all_words(blocks) == EXPECTED + + def test_list_item(self): + blocks = parse_html_string("") + hlist = next(b for b in blocks if isinstance(b, HList)) + item = list(hlist.items())[0] + assert all_words(item.blocks()) == EXPECTED + + def test_table_cell(self): + blocks = parse_html_string( + "
- hello world again
") + table = next(b for b in blocks if isinstance(b, Table)) + assert all_words(next(cell_blocks(table))) == EXPECTED + + def test_table_header_cell(self): + blocks = parse_html_string( + "
hello world again ") + table = next(b for b in blocks if isinstance(b, Table)) + assert all_words(next(cell_blocks(table))) == EXPECTED + + def test_blockquote(self): + blocks = parse_html_string("
hello world again hello world again") + quote = next(b for b in blocks if isinstance(b, Quote)) + assert all_words(quote.blocks()) == EXPECTED + + +class TestInlineRunsCoalesce: + """A run of inline content is one paragraph, not one per text node.""" + + def test_div_yields_a_single_paragraph(self): + blocks = parse_html_string("a b c") + paragraphs = [b for b in blocks if isinstance(b, Paragraph)] + assert len(paragraphs) == 1, f"expected one paragraph, got {len(blocks)} blocks" + assert words_of(paragraphs[0]) == ["a", "b", "c"] + + def test_cell_yields_a_single_paragraph(self): + blocks = parse_html_string("") + table = next(b for b in blocks if isinstance(b, Table)) + cell = next(cell_blocks(table)) + assert len(cell) == 1 + assert words_of(cell[0]) == ["a", "b", "c"] + + def test_block_child_splits_the_run(self): + """Inline runs either side of a block child stay separate, in order.""" + blocks = parse_html_string( + "
a b c ") + table = next(b for b in blocks if isinstance(b, Table)) + cell = next(cell_blocks(table)) + assert [words_of(b) for b in cell] == [["before"], ["middle"], ["after"]] + + def test_line_break_splits_the_run(self): + blocks = parse_html_string("
before middle
afterfirst") + paragraphs = [b for b in blocks if isinstance(b, Paragraph)] + assert [words_of(p) for p in paragraphs] == [["first"], ["second"]] + + def test_whitespace_between_blocks_makes_no_paragraph(self): + blocks = parse_html_string("
second\n") + assert [words_of(b) for b in blocks] == [["one"], ["two"]] + + +class TestLinksSurvive: + """ must produce LinkedWord wherever it appears.""" + + def test_link_in_cell(self): + blocks = parse_html_string( + 'one
\ntwo
\n') + table = next(b for b in blocks if isinstance(b, Table)) + cell = next(cell_blocks(table)) + found = [w for b in cell for w in getattr(b, 'words', [])] + + assert [w.text for w in found] == ["link", "text"] + linked = [w for w in found if isinstance(w, LinkedWord)] + assert len(linked) == 1 + assert linked[0].location == "http://x" + + def test_link_in_div(self): + blocks = parse_html_string('
link text see Section 2 now') + found = [w for b in blocks for w in getattr(b, 'words', [])] + assert [w.text for w in found] == ["see", "Section", "2", "now"] + assert all(isinstance(w, LinkedWord) for w in found[1:3]) + + def test_link_in_list_item(self): + blocks = parse_html_string('') + hlist = next(b for b in blocks if isinstance(b, HList)) + item = hlist._items[0] + found = [w for b in item.blocks() for w in getattr(b, 'words', [])] + assert [w.text for w in found] == ["click", "here"] + assert isinstance(found[0], LinkedWord) + + +class TestNestedContainers: + + def test_div_in_div(self): + blocks = parse_html_string("
- click here
outer") + assert [words_of(b) for b in blocks] == [["outer"], ["inner"], ["tail"]] + + def test_block_children_still_pass_through(self): + blocks = parse_html_string("innertail") + assert len(blocks) == 2 + assert words_of(blocks[0]) == ["Title"] + assert words_of(blocks[1]) == ["Body"] + + def test_cell_containing_a_list(self): + blocks = parse_html_string( + "Title
Body
") + table = next(b for b in blocks if isinstance(b, Table)) + cell = next(cell_blocks(table)) + assert isinstance(cell[0], Paragraph) + assert words_of(cell[0]) == ["intro"] + assert isinstance(cell[1], HList) + + +class TestComments: + + def test_comment_text_is_not_content(self): + blocks = parse_html_string("
intro
- item
realtext") + assert all_words(blocks) == ["real", "text"]