Inline tags map to ignore_handler because they are meant to be consumed by extract_text_content, but only paragraph_handler and heading_handler ever called it. Every other container walked its children calling process_element, so inline tags returned None and their text was dropped: <p>hello <b>world</b> again</p> -> hello world again (correct) <div>hello <b>world</b> again</div> -> nothing at all <li>hello <b>world</b> again</li> -> nothing at all <td>hello <b>world</b> again</td> -> nothing at all <td><a href=u>link</a> text</td> -> text (link discarded) div_handler ignored bare text nodes outright, so a div containing text produced no blocks whatsoever - which for real HTML and EPUB is most of the document. Where text did survive, in cells and list items, each text node became its own paragraph, so "a <b>b</b> c" fragmented onto separate lines. process_block_children now walks a container's children once, gathering runs of inline content into a single paragraph and letting block children through to their own handlers, preserving document order. div, li, td, th and blockquote all delegate to it, so they gain nested blocks, links and mixed content together. <br> ends the current run rather than being a no-op. extract_text_content is split so the run-level logic can be reused without building a synthetic element: extract_words_from_nodes takes the nodes directly, and skips comments, which previously had their text extracted as content. paragraph_handler keeps its own image-splitting path for now; folding it into process_block_children would also fix the ordering of text around images in a paragraph, but it carries the EPUB cover-detection behaviour and is left alone.
969 lines
33 KiB
Python
969 lines
33 KiB
Python
"""
|
|
HTML extraction module for converting HTML elements to pyWebLayout abstract elements.
|
|
|
|
This module provides handler functions for converting HTML elements into the abstract document structure
|
|
used by pyWebLayout, including paragraphs, headings, lists, tables, and inline formatting.
|
|
Each handler function has a robust signature that handles style hints, CSS classes, and attributes.
|
|
"""
|
|
|
|
from typing import List, Dict, Any, Optional, Union, Callable, Tuple, NamedTuple
|
|
from bs4 import BeautifulSoup, Tag, NavigableString
|
|
from bs4.element import CData, Comment, Doctype, ProcessingInstruction
|
|
from pyWebLayout.abstract.inline import Word
|
|
from pyWebLayout.abstract.block import (
|
|
Block,
|
|
Paragraph,
|
|
Heading,
|
|
HeadingLevel,
|
|
Quote,
|
|
CodeBlock,
|
|
HList,
|
|
ListItem,
|
|
ListStyle,
|
|
Table,
|
|
TableRow,
|
|
TableCell,
|
|
HorizontalRule,
|
|
Image,
|
|
)
|
|
from pyWebLayout.style import Font, FontWeight, FontStyle, TextDecoration
|
|
|
|
|
|
class StyleContext(NamedTuple):
|
|
"""
|
|
Immutable style context passed to handler functions.
|
|
Contains all styling information including inherited styles, CSS hints, and element attributes.
|
|
"""
|
|
|
|
font: Font
|
|
background: Optional[Tuple[int, int, int, int]]
|
|
css_classes: set
|
|
css_styles: Dict[str, str]
|
|
element_attributes: Dict[str, Any]
|
|
parent_elements: List[str] # Stack of parent element names
|
|
document: Optional[Any] # Reference to document for font registry
|
|
base_path: Optional[str] = None # Base path for resolving relative URLs
|
|
|
|
def with_font(self, font: Font) -> "StyleContext":
|
|
"""Create new context with modified font."""
|
|
return self._replace(font=font)
|
|
|
|
def with_background(
|
|
self, background: Optional[Tuple[int, int, int, int]]
|
|
) -> "StyleContext":
|
|
"""Create new context with modified background."""
|
|
return self._replace(background=background)
|
|
|
|
def with_css_classes(self, css_classes: set) -> "StyleContext":
|
|
"""Create new context with modified CSS classes."""
|
|
return self._replace(css_classes=css_classes)
|
|
|
|
def with_css_styles(self, css_styles: Dict[str, str]) -> "StyleContext":
|
|
"""Create new context with modified CSS styles."""
|
|
return self._replace(css_styles=css_styles)
|
|
|
|
def with_attributes(self, attributes: Dict[str, Any]) -> "StyleContext":
|
|
"""Create new context with modified element attributes."""
|
|
return self._replace(element_attributes=attributes)
|
|
|
|
def push_element(self, element_name: str) -> "StyleContext":
|
|
"""Create new context with element pushed onto parent stack."""
|
|
return self._replace(parent_elements=self.parent_elements + [element_name])
|
|
|
|
|
|
def create_base_context(
|
|
base_font: Optional[Font] = None,
|
|
document=None,
|
|
base_path: Optional[str] = None) -> StyleContext:
|
|
"""
|
|
Create a base style context with default values.
|
|
|
|
Args:
|
|
base_font: Base font to use, defaults to system default
|
|
document: Document instance for font registry
|
|
base_path: Base directory path for resolving relative URLs
|
|
|
|
Returns:
|
|
StyleContext with default values
|
|
"""
|
|
# Use document's font registry if available, otherwise create default font
|
|
if base_font is None:
|
|
if document and hasattr(document, 'get_or_create_font'):
|
|
base_font = document.get_or_create_font()
|
|
else:
|
|
base_font = Font()
|
|
|
|
return StyleContext(
|
|
font=base_font,
|
|
background=None,
|
|
css_classes=set(),
|
|
css_styles={},
|
|
element_attributes={},
|
|
parent_elements=[],
|
|
document=document,
|
|
base_path=base_path,
|
|
)
|
|
|
|
|
|
def apply_element_styling(context: StyleContext, element: Tag) -> StyleContext:
|
|
"""
|
|
Apply element-specific styling to context based on HTML element and attributes.
|
|
|
|
Args:
|
|
context: Current style context
|
|
element: BeautifulSoup Tag object
|
|
|
|
Returns:
|
|
New StyleContext with applied styling
|
|
"""
|
|
tag_name = element.name.lower()
|
|
attributes = dict(element.attrs) if element.attrs else {}
|
|
|
|
# Start with current context
|
|
new_context = context.with_attributes(attributes).push_element(tag_name)
|
|
|
|
# Apply CSS classes
|
|
css_classes = new_context.css_classes.copy()
|
|
if "class" in attributes:
|
|
classes = (
|
|
attributes["class"].split()
|
|
if isinstance(attributes["class"], str)
|
|
else attributes["class"]
|
|
)
|
|
css_classes.update(classes)
|
|
new_context = new_context.with_css_classes(css_classes)
|
|
|
|
# Apply inline styles
|
|
css_styles = new_context.css_styles.copy()
|
|
if "style" in attributes:
|
|
inline_styles = parse_inline_styles(attributes["style"])
|
|
css_styles.update(inline_styles)
|
|
new_context = new_context.with_css_styles(css_styles)
|
|
|
|
# Apply element-specific default styles
|
|
font = apply_element_font_styles(
|
|
new_context.font, tag_name, css_styles, new_context)
|
|
new_context = new_context.with_font(font)
|
|
|
|
# Apply background from styles
|
|
background = apply_background_styles(new_context.background, css_styles)
|
|
new_context = new_context.with_background(background)
|
|
|
|
return new_context
|
|
|
|
|
|
def parse_inline_styles(style_text: str) -> Dict[str, str]:
|
|
"""
|
|
Parse CSS inline styles into dictionary.
|
|
|
|
Args:
|
|
style_text: CSS style text (e.g., "color: red; font-weight: bold;")
|
|
|
|
Returns:
|
|
Dictionary of CSS property-value pairs
|
|
"""
|
|
styles = {}
|
|
for declaration in style_text.split(";"):
|
|
if ":" in declaration:
|
|
prop, value = declaration.split(":", 1)
|
|
styles[prop.strip().lower()] = value.strip()
|
|
return styles
|
|
|
|
|
|
def apply_element_font_styles(font: Font,
|
|
tag_name: str,
|
|
css_styles: Dict[str,
|
|
str],
|
|
context: Optional[StyleContext] = None) -> Font:
|
|
"""
|
|
Apply font styling based on HTML element and CSS styles.
|
|
Uses document's font registry when available to avoid creating duplicate fonts.
|
|
|
|
Args:
|
|
font: Current font
|
|
tag_name: HTML tag name
|
|
css_styles: CSS styles dictionary
|
|
context: Style context with document reference for font registry
|
|
|
|
Returns:
|
|
Font object with applied styling (either existing or newly created)
|
|
"""
|
|
# Default element styles
|
|
element_font_styles = {
|
|
"b": {"weight": FontWeight.BOLD},
|
|
"strong": {"weight": FontWeight.BOLD},
|
|
"i": {"style": FontStyle.ITALIC},
|
|
"em": {"style": FontStyle.ITALIC},
|
|
"u": {"decoration": TextDecoration.UNDERLINE},
|
|
"s": {"decoration": TextDecoration.STRIKETHROUGH},
|
|
"del": {"decoration": TextDecoration.STRIKETHROUGH},
|
|
"h1": {"size": 24, "weight": FontWeight.BOLD},
|
|
"h2": {"size": 20, "weight": FontWeight.BOLD},
|
|
"h3": {"size": 18, "weight": FontWeight.BOLD},
|
|
"h4": {"size": 16, "weight": FontWeight.BOLD},
|
|
"h5": {"size": 14, "weight": FontWeight.BOLD},
|
|
"h6": {"size": 12, "weight": FontWeight.BOLD},
|
|
}
|
|
|
|
# Start with current font properties
|
|
font_size = font.font_size
|
|
colour = font.colour
|
|
weight = font.weight
|
|
style = font.style
|
|
decoration = font.decoration
|
|
background = font.background
|
|
language = font.language
|
|
font_path = font._font_path
|
|
|
|
# Apply element default styles
|
|
if tag_name in element_font_styles:
|
|
elem_styles = element_font_styles[tag_name]
|
|
if "size" in elem_styles:
|
|
font_size = elem_styles["size"]
|
|
if "weight" in elem_styles:
|
|
weight = elem_styles["weight"]
|
|
if "style" in elem_styles:
|
|
style = elem_styles["style"]
|
|
if "decoration" in elem_styles:
|
|
decoration = elem_styles["decoration"]
|
|
|
|
# Apply CSS styles (override element defaults)
|
|
if "font-size" in css_styles:
|
|
# Parse font-size (simplified - could be enhanced)
|
|
size_value = css_styles["font-size"].lower()
|
|
if size_value.endswith("px"):
|
|
try:
|
|
font_size = int(float(size_value[:-2]))
|
|
except ValueError:
|
|
pass
|
|
elif size_value.endswith("pt"):
|
|
try:
|
|
font_size = int(float(size_value[:-2]))
|
|
except ValueError:
|
|
pass
|
|
|
|
if "font-weight" in css_styles:
|
|
weight_value = css_styles["font-weight"].lower()
|
|
if weight_value in ["bold", "700", "800", "900"]:
|
|
weight = FontWeight.BOLD
|
|
elif weight_value in ["normal", "400"]:
|
|
weight = FontWeight.NORMAL
|
|
|
|
if "font-style" in css_styles:
|
|
style_value = css_styles["font-style"].lower()
|
|
if style_value == "italic":
|
|
style = FontStyle.ITALIC
|
|
elif style_value == "normal":
|
|
style = FontStyle.NORMAL
|
|
|
|
if "text-decoration" in css_styles:
|
|
decoration_value = css_styles["text-decoration"].lower()
|
|
if "underline" in decoration_value:
|
|
decoration = TextDecoration.UNDERLINE
|
|
elif "line-through" in decoration_value:
|
|
decoration = TextDecoration.STRIKETHROUGH
|
|
elif "none" in decoration_value:
|
|
decoration = TextDecoration.NONE
|
|
|
|
if "color" in css_styles:
|
|
# Parse color (simplified - could be enhanced for hex, rgb, etc.)
|
|
color_value = css_styles["color"].lower()
|
|
color_map = {
|
|
"black": (0, 0, 0),
|
|
"white": (255, 255, 255),
|
|
"red": (255, 0, 0),
|
|
"green": (0, 255, 0),
|
|
"blue": (0, 0, 255),
|
|
}
|
|
if color_value in color_map:
|
|
colour = color_map[color_value]
|
|
elif color_value.startswith("#") and len(color_value) == 7:
|
|
try:
|
|
r = int(color_value[1:3], 16)
|
|
g = int(color_value[3:5], 16)
|
|
b = int(color_value[5:7], 16)
|
|
colour = (r, g, b)
|
|
except ValueError:
|
|
pass
|
|
|
|
# Use document's style registry if available to avoid creating duplicate styles
|
|
if context and context.document and hasattr(
|
|
context.document, 'get_or_create_style'):
|
|
# Create an abstract style first
|
|
from pyWebLayout.style.abstract_style import FontFamily, FontSize
|
|
|
|
# Map font properties to abstract style properties
|
|
font_family = FontFamily.SERIF # Default - could be enhanced to detect from font_path
|
|
if font_size:
|
|
font_size_value = font_size if isinstance(
|
|
font_size, int) else FontSize.MEDIUM
|
|
else:
|
|
font_size_value = FontSize.MEDIUM
|
|
|
|
# Create abstract style and register it
|
|
style_id, abstract_style = context.document.get_or_create_style(
|
|
font_family=font_family,
|
|
font_size=font_size_value,
|
|
font_weight=weight,
|
|
font_style=style,
|
|
text_decoration=decoration,
|
|
color=colour,
|
|
language=language
|
|
)
|
|
|
|
# Get the concrete font for this style
|
|
return context.document.get_font_for_style(abstract_style)
|
|
elif context and context.document and hasattr(context.document, 'get_or_create_font'):
|
|
# Fallback to old font registry system
|
|
return context.document.get_or_create_font(
|
|
font_path=font_path,
|
|
font_size=font_size,
|
|
colour=colour,
|
|
weight=weight,
|
|
style=style,
|
|
decoration=decoration,
|
|
background=background,
|
|
language=language,
|
|
min_hyphenation_width=font.min_hyphenation_width
|
|
)
|
|
else:
|
|
# Fallback to creating new font if no document context
|
|
return Font(
|
|
font_path=font_path,
|
|
font_size=font_size,
|
|
colour=colour,
|
|
weight=weight,
|
|
style=style,
|
|
decoration=decoration,
|
|
background=background,
|
|
language=language,
|
|
)
|
|
|
|
|
|
def apply_background_styles(
|
|
current_background: Optional[Tuple[int, int, int, int]], css_styles: Dict[str, str]
|
|
) -> Optional[Tuple[int, int, int, int]]:
|
|
"""
|
|
Apply background styling from CSS.
|
|
|
|
Args:
|
|
current_background: Current background color (RGBA)
|
|
css_styles: CSS styles dictionary
|
|
|
|
Returns:
|
|
New background color or None
|
|
"""
|
|
if "background-color" in css_styles:
|
|
bg_value = css_styles["background-color"].lower()
|
|
if bg_value == "transparent":
|
|
return None
|
|
# Add color parsing logic here if needed
|
|
|
|
return current_background
|
|
|
|
|
|
def extract_text_content(element: Tag, context: StyleContext) -> List[Word]:
|
|
"""
|
|
Extract text content from an element, handling inline formatting and links.
|
|
|
|
Args:
|
|
element: BeautifulSoup Tag object
|
|
context: Current style context
|
|
|
|
Returns:
|
|
List of Word objects (including LinkedWord for hyperlinks)
|
|
"""
|
|
return extract_words_from_nodes(list(element.children), context)
|
|
|
|
|
|
def extract_words_from_nodes(nodes: List, context: StyleContext) -> List[Word]:
|
|
"""
|
|
Extract words from a sequence of sibling nodes.
|
|
|
|
Separated from extract_text_content so that a container holding a mix of
|
|
inline and block children can hand over just the inline runs, without
|
|
building a synthetic element to wrap them in.
|
|
|
|
Args:
|
|
nodes: BeautifulSoup nodes (Tags and NavigableStrings) in document order
|
|
context: Current style context
|
|
|
|
Returns:
|
|
List of Word objects (including LinkedWord for hyperlinks)
|
|
"""
|
|
from pyWebLayout.abstract.inline import LinkedWord
|
|
from pyWebLayout.abstract.functional import LinkType
|
|
|
|
words = []
|
|
|
|
for child in nodes:
|
|
# Comments and processing instructions are NavigableString subclasses;
|
|
# their text is markup, not content.
|
|
if isinstance(child, (Comment, Doctype, CData, ProcessingInstruction)):
|
|
continue
|
|
|
|
if isinstance(child, NavigableString):
|
|
# Plain text - split into words
|
|
text = str(child).strip()
|
|
if text:
|
|
word_texts = text.split()
|
|
for word_text in word_texts:
|
|
if word_text:
|
|
words.append(Word(word_text, context.font, context.background))
|
|
elif isinstance(child, Tag):
|
|
# Special handling for <a> tags (hyperlinks)
|
|
if child.name.lower() == "a":
|
|
href = child.get('href', '')
|
|
if href:
|
|
# Determine link type based on href
|
|
if href.startswith(('http://', 'https://')):
|
|
link_type = LinkType.EXTERNAL
|
|
elif href.startswith('#'):
|
|
link_type = LinkType.INTERNAL
|
|
elif href.startswith('javascript:') or href.startswith('api:'):
|
|
link_type = LinkType.API
|
|
else:
|
|
link_type = LinkType.INTERNAL
|
|
|
|
# Apply link styling
|
|
child_context = apply_element_styling(context, child)
|
|
|
|
# Extract text and create LinkedWord for each word
|
|
link_text = child.get_text(strip=True)
|
|
title = child.get('title', '')
|
|
|
|
for word_text in link_text.split():
|
|
if word_text:
|
|
linked_word = LinkedWord(
|
|
text=word_text,
|
|
style=child_context.font,
|
|
location=href,
|
|
link_type=link_type,
|
|
background=child_context.background,
|
|
title=title if title else None
|
|
)
|
|
words.append(linked_word)
|
|
else:
|
|
# <a> without href - treat as normal text
|
|
child_context = apply_element_styling(context, child)
|
|
child_words = extract_text_content(child, child_context)
|
|
words.extend(child_words)
|
|
|
|
# Process other inline elements
|
|
elif child.name.lower() in [
|
|
"span",
|
|
"strong",
|
|
"b",
|
|
"em",
|
|
"i",
|
|
"u",
|
|
"s",
|
|
"del",
|
|
"ins",
|
|
"mark",
|
|
"small",
|
|
"sub",
|
|
"sup",
|
|
"code",
|
|
"q",
|
|
"cite",
|
|
"abbr",
|
|
"time",
|
|
]:
|
|
child_context = apply_element_styling(context, child)
|
|
child_words = extract_text_content(child, child_context)
|
|
words.extend(child_words)
|
|
else:
|
|
# Block element - shouldn't happen in well-formed HTML but handle
|
|
# gracefully
|
|
child_context = apply_element_styling(context, child)
|
|
child_result = process_element(child, child_context)
|
|
if isinstance(child_result, list):
|
|
for block in child_result:
|
|
if isinstance(block, Paragraph):
|
|
for _, word in block.words_iter():
|
|
words.append(word)
|
|
elif isinstance(child_result, Paragraph):
|
|
for _, word in child_result.words_iter():
|
|
words.append(word)
|
|
|
|
return words
|
|
|
|
|
|
# Tags that flow within a line of text rather than forming a block of their own.
|
|
# They carry no handler of their own: extract_words_from_nodes consumes them,
|
|
# applying their styling to the words they contain.
|
|
INLINE_TAGS = frozenset({
|
|
"a", "span", "strong", "b", "em", "i", "u", "s", "del", "ins", "mark",
|
|
"small", "sub", "sup", "code", "q", "cite", "abbr", "time",
|
|
})
|
|
|
|
|
|
def is_inline(node) -> bool:
|
|
"""
|
|
Whether a node belongs to a run of text rather than standing as its own block.
|
|
|
|
Args:
|
|
node: A BeautifulSoup Tag or NavigableString
|
|
|
|
Returns:
|
|
True for text and inline tags, False for block-level tags
|
|
"""
|
|
if isinstance(node, Tag):
|
|
return node.name.lower() in INLINE_TAGS
|
|
if isinstance(node, (Comment, Doctype, CData, ProcessingInstruction)):
|
|
return False
|
|
return isinstance(node, NavigableString)
|
|
|
|
|
|
def process_block_children(element: Tag, context: StyleContext) -> List[Block]:
|
|
"""
|
|
Process a container's children into a list of blocks.
|
|
|
|
Containers may hold a mix of inline and block content. Consecutive inline
|
|
children are gathered into a run and become one Paragraph; a block child ends
|
|
the current run and is processed by its own handler. This is the single entry
|
|
point for every container that is not itself a paragraph - div, li, td, th,
|
|
blockquote and the semantic containers.
|
|
|
|
Without this, inline tags reach process_element, whose handler for them is
|
|
ignore_handler, and their text is silently dropped.
|
|
|
|
Args:
|
|
element: The container element
|
|
context: Current style context
|
|
|
|
Returns:
|
|
Blocks in document order
|
|
"""
|
|
blocks: List[Block] = []
|
|
run: List = []
|
|
|
|
def flush_run():
|
|
"""Turn the pending inline run into a paragraph, if it holds any words."""
|
|
if not run:
|
|
return
|
|
words = extract_words_from_nodes(run, context)
|
|
run.clear()
|
|
if words:
|
|
paragraph = Paragraph(context.font)
|
|
for word in words:
|
|
paragraph.add_word(word)
|
|
blocks.append(paragraph)
|
|
|
|
for child in element.children:
|
|
# <br> ends the current line of text and starts a new one.
|
|
if isinstance(child, Tag) and child.name.lower() == "br":
|
|
flush_run()
|
|
continue
|
|
|
|
if is_inline(child):
|
|
run.append(child)
|
|
continue
|
|
|
|
if not isinstance(child, Tag):
|
|
continue # comments and similar
|
|
|
|
flush_run()
|
|
child_context = apply_element_styling(context, child)
|
|
result = process_element(child, child_context)
|
|
if result:
|
|
if isinstance(result, list):
|
|
blocks.extend(result)
|
|
else:
|
|
blocks.append(result)
|
|
|
|
flush_run()
|
|
return blocks
|
|
|
|
|
|
def process_element(
|
|
element: Tag, context: StyleContext
|
|
) -> Union[Block, List[Block], None]:
|
|
"""
|
|
Process a single HTML element using appropriate handler.
|
|
|
|
Args:
|
|
element: BeautifulSoup Tag object
|
|
context: Current style context
|
|
|
|
Returns:
|
|
Block object(s) or None if element should be ignored
|
|
"""
|
|
tag_name = element.name.lower()
|
|
handler = HANDLERS.get(tag_name, generic_handler)
|
|
return handler(element, context)
|
|
|
|
|
|
# Handler function signatures:
|
|
# All handlers receive (element: Tag, context: StyleContext) ->
|
|
# Union[Block, List[Block], None]
|
|
|
|
|
|
def paragraph_handler(element: Tag, context: StyleContext) -> Union[Paragraph, List[Block], Image]:
|
|
"""
|
|
Handle <p> elements.
|
|
|
|
Special handling for paragraphs containing images:
|
|
- If the paragraph contains only an image (common in EPUBs), return the image block
|
|
- If the paragraph contains images mixed with text, split into separate blocks
|
|
- Otherwise, return a normal paragraph with text content
|
|
"""
|
|
# Check if paragraph contains any img tags (including nested ones)
|
|
img_tags = element.find_all('img')
|
|
|
|
if img_tags:
|
|
# Paragraph contains images - need special handling
|
|
blocks = []
|
|
|
|
# Check if this is an image-only paragraph (very common in EPUBs)
|
|
# Get text content without the img tags
|
|
text_content = element.get_text(strip=True)
|
|
|
|
if not text_content or len(text_content.strip()) == 0:
|
|
# Image-only paragraph - return just the image(s)
|
|
for img_tag in img_tags:
|
|
child_context = apply_element_styling(context, img_tag)
|
|
img_block = image_handler(img_tag, child_context)
|
|
if img_block:
|
|
blocks.append(img_block)
|
|
|
|
# Return single image or list of images
|
|
if len(blocks) == 1:
|
|
return blocks[0]
|
|
return blocks if blocks else Paragraph(context.font)
|
|
|
|
# Mixed content - paragraph has both text and images
|
|
# Process children in order to preserve structure
|
|
for child in element.children:
|
|
if isinstance(child, Tag):
|
|
if child.name == 'img':
|
|
# Add the image as a separate block
|
|
child_context = apply_element_styling(context, child)
|
|
img_block = image_handler(child, child_context)
|
|
if img_block:
|
|
blocks.append(img_block)
|
|
else:
|
|
# Process other inline elements as part of text
|
|
# This will be handled by extract_text_content below
|
|
pass
|
|
|
|
# Also add a paragraph with the text content
|
|
paragraph = Paragraph(context.font)
|
|
words = extract_text_content(element, context)
|
|
if words:
|
|
for word in words:
|
|
paragraph.add_word(word)
|
|
blocks.insert(0, paragraph) # Text comes before images
|
|
|
|
return blocks if blocks else Paragraph(context.font)
|
|
|
|
# No images - normal paragraph handling
|
|
paragraph = Paragraph(context.font)
|
|
words = extract_text_content(element, context)
|
|
for word in words:
|
|
paragraph.add_word(word)
|
|
return paragraph
|
|
|
|
|
|
def div_handler(element: Tag, context: StyleContext) -> List[Block]:
|
|
"""Handle <div> elements - treat as generic container."""
|
|
return process_block_children(element, context)
|
|
|
|
|
|
def heading_handler(element: Tag, context: StyleContext) -> Heading:
|
|
"""Handle <h1>-<h6> elements."""
|
|
level_map = {
|
|
"h1": HeadingLevel.H1,
|
|
"h2": HeadingLevel.H2,
|
|
"h3": HeadingLevel.H3,
|
|
"h4": HeadingLevel.H4,
|
|
"h5": HeadingLevel.H5,
|
|
"h6": HeadingLevel.H6,
|
|
}
|
|
|
|
level = level_map.get(element.name.lower(), HeadingLevel.H1)
|
|
heading = Heading(level, context.font)
|
|
words = extract_text_content(element, context)
|
|
for word in words:
|
|
heading.add_word(word)
|
|
return heading
|
|
|
|
|
|
def blockquote_handler(element: Tag, context: StyleContext) -> Quote:
|
|
"""Handle <blockquote> elements."""
|
|
quote = Quote(context.font)
|
|
for block in process_block_children(element, context):
|
|
quote.add_block(block)
|
|
return quote
|
|
|
|
|
|
def preformatted_handler(element: Tag, context: StyleContext) -> CodeBlock:
|
|
"""Handle <pre> elements."""
|
|
language = context.element_attributes.get("data-language", "")
|
|
code_block = CodeBlock(language)
|
|
|
|
# Preserve whitespace and line breaks in preformatted text
|
|
text = element.get_text(separator="\n", strip=False)
|
|
for line in text.split("\n"):
|
|
code_block.add_line(line)
|
|
|
|
return code_block
|
|
|
|
|
|
def code_handler(element: Tag, context: StyleContext) -> Union[CodeBlock, None]:
|
|
"""Handle <code> elements."""
|
|
# If parent is <pre>, this is handled by preformatted_handler
|
|
if context.parent_elements and context.parent_elements[-1] == "pre":
|
|
return None # Will be handled by parent
|
|
|
|
# Inline code - handled during text extraction
|
|
return None
|
|
|
|
|
|
def unordered_list_handler(element: Tag, context: StyleContext) -> HList:
|
|
"""Handle <ul> elements."""
|
|
hlist = HList(ListStyle.UNORDERED, context.font)
|
|
for child in element.children:
|
|
if isinstance(child, Tag) and child.name.lower() == "li":
|
|
child_context = apply_element_styling(context, child)
|
|
item = process_element(child, child_context)
|
|
if item:
|
|
hlist.add_item(item)
|
|
return hlist
|
|
|
|
|
|
def ordered_list_handler(element: Tag, context: StyleContext) -> HList:
|
|
"""Handle <ol> elements."""
|
|
hlist = HList(ListStyle.ORDERED, context.font)
|
|
for child in element.children:
|
|
if isinstance(child, Tag) and child.name.lower() == "li":
|
|
child_context = apply_element_styling(context, child)
|
|
item = process_element(child, child_context)
|
|
if item:
|
|
hlist.add_item(item)
|
|
return hlist
|
|
|
|
|
|
def list_item_handler(element: Tag, context: StyleContext) -> ListItem:
|
|
"""Handle <li> elements."""
|
|
list_item = ListItem(None, context.font)
|
|
for block in process_block_children(element, context):
|
|
list_item.add_block(block)
|
|
return list_item
|
|
|
|
|
|
def table_handler(element: Tag, context: StyleContext) -> Table:
|
|
"""Handle <table> elements."""
|
|
caption = None
|
|
caption_elem = element.find("caption")
|
|
if caption_elem:
|
|
caption = caption_elem.get_text(strip=True)
|
|
|
|
table = Table(caption, context.font)
|
|
|
|
# Process table rows
|
|
for child in element.children:
|
|
if isinstance(child, Tag):
|
|
if child.name.lower() == "tr":
|
|
child_context = apply_element_styling(context, child)
|
|
row = process_element(child, child_context)
|
|
if row:
|
|
table.add_row(row)
|
|
elif child.name.lower() in ["thead", "tbody", "tfoot"]:
|
|
section = "header" if child.name.lower() == "thead" else "body"
|
|
section = "footer" if child.name.lower() == "tfoot" else section
|
|
|
|
for row_elem in child.find_all("tr"):
|
|
child_context = apply_element_styling(context, row_elem)
|
|
row = process_element(row_elem, child_context)
|
|
if row:
|
|
table.add_row(row, section)
|
|
|
|
return table
|
|
|
|
|
|
def table_row_handler(element: Tag, context: StyleContext) -> TableRow:
|
|
"""Handle <tr> elements."""
|
|
row = TableRow(context.font)
|
|
for child in element.children:
|
|
if isinstance(child, Tag) and child.name.lower() in ["td", "th"]:
|
|
child_context = apply_element_styling(context, child)
|
|
cell = process_element(child, child_context)
|
|
if cell:
|
|
row.add_cell(cell)
|
|
return row
|
|
|
|
|
|
def table_cell_handler(element: Tag, context: StyleContext) -> TableCell:
|
|
"""Handle <td> elements."""
|
|
colspan = int(context.element_attributes.get("colspan", 1))
|
|
rowspan = int(context.element_attributes.get("rowspan", 1))
|
|
cell = TableCell(False, colspan, rowspan, context.font)
|
|
|
|
for block in process_block_children(element, context):
|
|
cell.add_block(block)
|
|
|
|
return cell
|
|
|
|
|
|
def table_header_cell_handler(element: Tag, context: StyleContext) -> TableCell:
|
|
"""Handle <th> elements."""
|
|
colspan = int(context.element_attributes.get("colspan", 1))
|
|
rowspan = int(context.element_attributes.get("rowspan", 1))
|
|
cell = TableCell(True, colspan, rowspan, context.font)
|
|
|
|
for block in process_block_children(element, context):
|
|
cell.add_block(block)
|
|
|
|
return cell
|
|
|
|
|
|
def horizontal_rule_handler(element: Tag, context: StyleContext) -> HorizontalRule:
|
|
"""Handle <hr> elements."""
|
|
return HorizontalRule()
|
|
|
|
|
|
def line_break_handler(element: Tag, context: StyleContext) -> None:
|
|
"""Handle <br> elements."""
|
|
# Line breaks are typically handled at the paragraph level
|
|
return None
|
|
|
|
|
|
def image_handler(element: Tag, context: StyleContext) -> Image:
|
|
"""Handle <img> elements."""
|
|
import os
|
|
import urllib.parse
|
|
|
|
src = context.element_attributes.get("src", "")
|
|
alt_text = context.element_attributes.get("alt", "")
|
|
|
|
# Resolve relative paths if base_path is provided
|
|
if context.base_path and src and not src.startswith(('http://', 'https://', '/')):
|
|
# Parse the src to handle URL-encoded characters
|
|
src_decoded = urllib.parse.unquote(src)
|
|
# Resolve relative path to absolute path
|
|
src = os.path.normpath(os.path.join(context.base_path, src_decoded))
|
|
|
|
# Parse dimensions if provided
|
|
width = height = None
|
|
try:
|
|
if "width" in context.element_attributes:
|
|
width = int(context.element_attributes["width"])
|
|
if "height" in context.element_attributes:
|
|
height = int(context.element_attributes["height"])
|
|
except ValueError:
|
|
pass
|
|
|
|
return Image(source=src, alt_text=alt_text, width=width, height=height)
|
|
|
|
|
|
def ignore_handler(element: Tag, context: StyleContext) -> None:
|
|
"""Handle elements that should be ignored."""
|
|
return None
|
|
|
|
|
|
def generic_handler(element: Tag, context: StyleContext) -> List[Block]:
|
|
"""Handle unknown elements as generic containers."""
|
|
return div_handler(element, context)
|
|
|
|
|
|
# Handler registry - maps HTML tag names to handler functions
|
|
HANDLERS: Dict[str, Callable[[Tag, StyleContext], Union[Block, List[Block], None]]] = {
|
|
# Block elements
|
|
"p": paragraph_handler,
|
|
"div": div_handler,
|
|
"h1": heading_handler,
|
|
"h2": heading_handler,
|
|
"h3": heading_handler,
|
|
"h4": heading_handler,
|
|
"h5": heading_handler,
|
|
"h6": heading_handler,
|
|
"blockquote": blockquote_handler,
|
|
"pre": preformatted_handler,
|
|
"code": code_handler,
|
|
"ul": unordered_list_handler,
|
|
"ol": ordered_list_handler,
|
|
"li": list_item_handler,
|
|
"table": table_handler,
|
|
"tr": table_row_handler,
|
|
"td": table_cell_handler,
|
|
"th": table_header_cell_handler,
|
|
"hr": horizontal_rule_handler,
|
|
"br": line_break_handler,
|
|
# Semantic elements (treated as containers)
|
|
"section": div_handler,
|
|
"article": div_handler,
|
|
"aside": div_handler,
|
|
"nav": div_handler,
|
|
"header": div_handler,
|
|
"footer": div_handler,
|
|
"main": div_handler,
|
|
"figure": div_handler,
|
|
"figcaption": paragraph_handler,
|
|
# Media elements
|
|
"img": image_handler,
|
|
# Inline elements (handled during text extraction)
|
|
"span": ignore_handler,
|
|
"a": ignore_handler,
|
|
"strong": ignore_handler,
|
|
"b": ignore_handler,
|
|
"em": ignore_handler,
|
|
"i": ignore_handler,
|
|
"u": ignore_handler,
|
|
"s": ignore_handler,
|
|
"del": ignore_handler,
|
|
"ins": ignore_handler,
|
|
"mark": ignore_handler,
|
|
"small": ignore_handler,
|
|
"sub": ignore_handler,
|
|
"sup": ignore_handler,
|
|
"q": ignore_handler,
|
|
"cite": ignore_handler,
|
|
"abbr": ignore_handler,
|
|
"time": ignore_handler,
|
|
# Ignored elements
|
|
"script": ignore_handler,
|
|
"style": ignore_handler,
|
|
"meta": ignore_handler,
|
|
"link": ignore_handler,
|
|
"head": ignore_handler,
|
|
"title": ignore_handler,
|
|
}
|
|
|
|
|
|
def parse_html_string(
|
|
html_string: str, base_font: Optional[Font] = None, document=None, base_path: Optional[str] = None
|
|
) -> List[Block]:
|
|
"""
|
|
Parse HTML string and return list of Block objects.
|
|
|
|
Args:
|
|
html_string: HTML content to parse
|
|
base_font: Base font for styling, defaults to system default
|
|
document: Document instance for font registry to avoid duplicate fonts
|
|
base_path: Base directory path for resolving relative URLs (e.g., image sources)
|
|
|
|
Returns:
|
|
List of Block objects representing the document structure
|
|
"""
|
|
soup = BeautifulSoup(html_string, "html.parser")
|
|
context = create_base_context(base_font, document, base_path)
|
|
|
|
blocks = []
|
|
|
|
# Process the body if it exists, otherwise process all top-level elements
|
|
root_element = soup.find("body") or soup
|
|
|
|
for element in root_element.children:
|
|
if isinstance(element, Tag):
|
|
element_context = apply_element_styling(context, element)
|
|
result = process_element(element, element_context)
|
|
if result:
|
|
if isinstance(result, list):
|
|
blocks.extend(result)
|
|
else:
|
|
blocks.append(result)
|
|
|
|
return blocks
|