diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py index 67e87250d..de4ea6d1d 100644 --- a/crawl4ai/content_scraping_strategy.py +++ b/crawl4ai/content_scraping_strategy.py @@ -729,8 +729,10 @@ def _scrap( elif content_element is None: content_element = body - # Replace mermaid SVGs with text before they get stripped - for svg in body.xpath('.//svg[starts-with(@id, "mermaid-")]'): + # Replace mermaid SVGs with text before they get stripped. Runs on + # content_element (see the note above the cleanup block below) so a + # selector picks up the placeholder instead of a stale, unmutated copy. + for svg in content_element.xpath('.//svg[starts-with(@id, "mermaid-")]'): try: diagram_type = svg.get("aria-roledescription", "diagram") labels = [] @@ -789,14 +791,15 @@ def _scrap( except Exception: pass - # Remove script and style tags + # Remove script and style tags. Same reason as the mermaid pass + # above: this must land on content_element, not body. for tag in ["style", "link", "meta", "noscript"]: - for element in body.xpath(f".//{tag}"): + for element in content_element.xpath(f".//{tag}"): if element.getparent() is not None: element.getparent().remove(element) - + # Handle script separately - for element in body.xpath(f".//script"): + for element in content_element.xpath(f".//script"): parent = element.getparent() if parent is not None: tail = element.tail # Get the tail text @@ -824,13 +827,16 @@ def _scrap( ) kwargs["exclude_domains"].update(kwargs["exclude_social_media_domains"]) - # Process forms if needed + # Process forms if needed. Same reason as the passes above: this + # must land on content_element, not body. if kwargs.get("remove_forms", False): - for form in body.xpath(".//form"): + for form in content_element.xpath(".//form"): if form.getparent() is not None: form.getparent().remove(form) - # Process content + # Process content. Link and media collection stays page-wide by + # design, so this deliberately reads from body, not content_element, + # even when a selector is in play. media = {"images": [], "videos": [], "audios": [], "tables": []} internal_links_dict = {} external_links_dict = {} @@ -846,7 +852,8 @@ def _scrap( **kwargs, ) - # Extract tables using the table extraction strategy if provided + # Extract tables using the table extraction strategy if provided. + # Page-wide for the same reason as the link/media pass above. if 'table' not in excluded_tags: table_extraction = kwargs.get('table_extraction') if table_extraction: @@ -857,10 +864,16 @@ def _scrap( extracted_tables = table_extraction.extract_tables(body, **kwargs) media["tables"].extend(extracted_tables) + # Every pass below mutates and then serialises content_element, the + # thing that becomes cleaned_html. With a css_selector or + # target_elements that is the deep copy severed from body at the + # top of this method; without one content_element is body itself, + # so the common path is unchanged. + # Handle only_text option if kwargs.get("only_text", False): for tag in ONLY_TEXT_ELIGIBLE_TAGS: - for element in body.xpath(f".//{tag}"): + for element in content_element.xpath(f".//{tag}"): if element.text: new_text = lhtml.Element("span") new_text.text = element.text_content() @@ -868,17 +881,18 @@ def _scrap( element.getparent().replace(element, new_text) # Clean base64 images - for img in body.xpath(".//img[@src]"): + for img in content_element.xpath(".//img[@src]"): src = img.get("src", "") if self.BASE64_PATTERN.match(src): img.set("src", self.BASE64_PATTERN.sub("", src)) # Remove empty elements - self.remove_empty_elements_fast(body, 1) + self.remove_empty_elements_fast(content_element, 1) # Remove unneeded attributes self.remove_unwanted_attributes_fast( - body, keep_data_attributes=kwargs.get("keep_data_attributes", False) + content_element, + keep_data_attributes=kwargs.get("keep_data_attributes", False), ) # Generate output HTML diff --git a/tests/test_selector_post_processing.py b/tests/test_selector_post_processing.py new file mode 100644 index 000000000..4a9d9e835 --- /dev/null +++ b/tests/test_selector_post_processing.py @@ -0,0 +1,146 @@ +"""Tests for post-processing when css_selector or target_elements is set. + +The bug: _scrap() deep-copies the selector match into a new content_element, +then ran only_text, base64 image cleanup, empty-element removal and attribute +stripping, plus mermaid SVG replacement, style/link/meta/noscript/script +removal and form removal, against `body`. The copy is detached from `body`, +so none of those passes reached the HTML that is serialised into +cleaned_html: survived only_text, inline style, onclick, data-* and whole +base64 payloads were emitted verbatim, and a + + + + + +""" + +COMMON = dict(url="raw://test", html=SAMPLE_HTML, only_text=True) + +SELECTORS = [ + pytest.param({}, id="no-selector"), + pytest.param({"css_selector": ".job"}, id="css_selector"), + pytest.param({"target_elements": [".job"]}, id="target_elements"), + pytest.param({"css_selector": "body", "target_elements": [".job"]}, id="both"), +] + + +@pytest.fixture +def scraper(): + return LXMLWebScrapingStrategy() + + +@pytest.mark.parametrize("selector", SELECTORS) +class TestPostProcessingRunsWithSelector: + def test_only_text_unwraps_inline_tags(self, scraper, selector): + """only_text should unwrap / whether or not a selector is set.""" + cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"] + assert "" not in cleaned + assert "" not in cleaned + assert "Postgres" in cleaned + assert "Python" in cleaned + + def test_base64_image_src_is_truncated(self, scraper, selector): + """The base64 payload should never reach cleaned_html.""" + cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"] + assert "base64" not in cleaned + assert 'src=""' in cleaned + + def test_empty_elements_are_removed(self, scraper, selector): + """The empty
should be dropped.""" + cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"] + assert "tracker" not in cleaned + + def test_unwanted_attributes_are_stripped(self, scraper, selector): + """style/onclick/data-* go, class/href stay.""" + cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"] + assert "style=" not in cleaned + assert "onclick=" not in cleaned + assert "data-tracking" not in cleaned + assert 'class="job"' in cleaned + assert 'href="/apply"' in cleaned + + def test_style_script_noscript_are_removed(self, scraper, selector): + """style/script/noscript inside the selection must not survive.""" + cleaned = scraper._scrap(**COMMON, **selector)["cleaned_html"] + assert "