"""

Tests for pages whose body ends up nested inside an unclosed <noscript>.

A page that nests <noscript> (for example a lazy-load plugin wrapping Google Tag

Manager's own noscript) loses its outer closing tag when a scripting-enabled

browser serializes it. Re-parsed by lxml, the rest of the document becomes a

child of the still-open <noscript>, and the scraper's noscript removal used to

delete the entire page, leaving cleaned_html with an empty body.

"""

import pytest

from crawl4ai.content_scraping_strategy import (

LXMLWebScrapingStrategy,

unwrap_orphaned_noscript,

)

# Shape of page.content() for such a page: two opening tags, one closing tag.

DAMAGED_CAPTURE = """

<html>

<head><title>Example shop</title></head>

<body>

<noscript><iframe src="about:blank" height="0" width="0"></iframe><noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-TEST" height="0" width="0"></iframe></noscript>

<div class="site-content">

<h1>Unstunned. Hand slaughtered. Halal certified.</h1>

<p>Our entire assortment is strictly halal, prepared by qualified staff.</p>

</div>

</body>

</html>

"""

@pytest.fixture

def strategy():

return LXMLWebScrapingStrategy()

def test_body_nested_in_unclosed_noscript_is_kept(strategy):

result = strategy.scrap("https://example.com/", DAMAGED_CAPTURE)

assert "Unstunned. Hand slaughtered. Halal certified." in result.cleaned_html

assert "strictly halal" in result.cleaned_html

def test_balanced_noscript_is_still_removed(strategy):

html = (

"<html><body>"

"<noscript><p>Please enable JavaScript</p></noscript>"

"<p>Real page copy that should be kept.</p>"

"</body></html>"

)

result = strategy.scrap("https://example.com/", html)

assert "Please enable JavaScript" not in result.cleaned_html

assert "Real page copy that should be kept." in result.cleaned_html

def test_unwrap_leaves_well_formed_documents_untouched():

html = "<html><body><noscript><p>fallback</p></noscript><p>copy</p></body></html>"

assert unwrap_orphaned_noscript(html) == html

def test_unwrap_removes_tags_but_keeps_contents():

unwrapped = unwrap_orphaned_noscript(DAMAGED_CAPTURE)

assert "noscript" not in unwrapped

assert "<h1>Unstunned. Hand slaughtered. Halal certified.</h1>" in unwrapped

def test_fit_html_keeps_body_nested_in_unclosed_noscript():

from crawl4ai.utils import preprocess_html_for_schema

fit_html = preprocess_html_for_schema(DAMAGED_CAPTURE, text_threshold=500, max_size=300_000)

assert "Unstunned. Hand slaughtered. Halal certified." in fit_html

@pytest.mark.parametrize(

"decoy",

[

'<script>const sample = "<noscript>";</script>',

"<style>/* <noscript> */</style>",

"<!-- <noscript> -->",

],

)

def test_noscript_text_in_script_style_or_comment_is_not_counted(strategy, decoy):

html = (

"<html><body>"

f"{decoy}"

"<noscript>Please enable JavaScript</noscript>"

"<h1>Real heading</h1>"

"</body></html>"

)

result = strategy.scrap("https://example.com/", html)

assert "Please enable JavaScript" not in result.cleaned_html

assert "Real heading" in result.cleaned_html

@pytest.mark.parametrize("opener", ["<!--", "<script>", "<style>"])

def test_many_unclosed_raw_text_openers_stay_fast(opener):

import time

html = (opener + "a" * 42) * 20_000 # about 1 MB

start = time.time()

unwrap_orphaned_noscript(html)

# Quadratic matching took over a minute here; linear takes milliseconds.

assert time.time() - start < 2