#!/usr/bin/env python3

"""

Prefetch Mode and Two-Phase Crawling Example

Prefetch mode is a fast path that skips heavy processing and returns

only HTML + links. This is ideal for:

- Site mapping: Quickly discover all URLs

- Selective crawling: Find URLs first, then process only what you need

- Link validation: Check which pages exist without full processing

- Crawl planning: Estimate size before committing resources

Key concept:

- `prefetch=True` in CrawlerRunConfig enables fast link-only extraction

- Skips: markdown generation, content scraping, media extraction, LLM extraction

- Returns: HTML and links dictionary

Performance benefit: ~5-10x faster than full processing

"""

import asyncio

import time

from typing import List, Dict

from crawl4ai import AsyncWebCrawler, CrawlerRunConfig

async def example_basic_prefetch():

"""

Example 1: Basic prefetch mode.

Shows how prefetch returns HTML and links without heavy processing.

"""

print("\n" + "=" * 60)

print("Example 1: Basic Prefetch Mode")

print("=" * 60)

async with AsyncWebCrawler(verbose=False) as crawler:

# Enable prefetch mode

config = CrawlerRunConfig(prefetch=True)

print("\nFetching with prefetch=True...")

result = await crawler.arun("https://books.toscrape.com", config=config)

print(f"\nResult summary:")

print(f" Success: {result.success}")

print(f" HTML length: {len(result.html) if result.html else 0} chars")

print(f" Internal links: {len(result.links.get('internal', []))}")

print(f" External links: {len(result.links.get('external', []))}")

# These should be None/empty in prefetch mode

print(f"\n Skipped processing:")

print(f" Markdown: {result.markdown}")

print(f" Cleaned HTML: {result.cleaned_html}")

print(f" Extracted content: {result.extracted_content}")

# Show some discovered links

internal_links = result.links.get("internal", [])

if internal_links:

print(f"\n Sample internal links:")

for link in internal_links[:5]:

print(f" - {link['href'][:60]}...")

async def example_performance_comparison():

"""

Example 2: Compare prefetch vs full processing performance.

"""

print("\n" + "=" * 60)

print("Example 2: Performance Comparison")

print("=" * 60)

url = "https://books.toscrape.com"

async with AsyncWebCrawler(verbose=False) as crawler:

# Warm up - first request is slower due to browser startup

await crawler.arun(url, config=CrawlerRunConfig())

# Prefetch mode timing

start = time.time()

prefetch_result = await crawler.arun(url, config=CrawlerRunConfig(prefetch=True))

prefetch_time = time.time() - start

# Full processing timing

start = time.time()

full_result = await crawler.arun(url, config=CrawlerRunConfig())

full_time = time.time() - start

print(f"\nTiming comparison:")

print(f" Prefetch mode: {prefetch_time:.3f}s")

print(f" Full processing: {full_time:.3f}s")

print(f" Speedup: {full_time / prefetch_time:.1f}x faster")

print(f"\nOutput comparison:")

print(f" Prefetch - Links found: {len(prefetch_result.links.get('internal', []))}")

print(f" Full - Links found: {len(full_result.links.get('internal', []))}")

print(f" Full - Markdown length: {len(full_result.markdown.raw_markdown) if full_result.markdown else 0}")

async def example_two_phase_crawl():

"""

Example 3: Two-phase crawling pattern.

Phase 1: Fast discovery with prefetch

Phase 2: Full processing on selected URLs

"""

print("\n" + "=" * 60)

print("Example 3: Two-Phase Crawling")

print("=" * 60)

async with AsyncWebCrawler(verbose=False) as crawler:

# ═══════════════════════════════════════════════════════════

# Phase 1: Fast URL discovery

# ═══════════════════════════════════════════════════════════

print("\n--- Phase 1: Fast Discovery ---")

prefetch_config = CrawlerRunConfig(prefetch=True)

start = time.time()

discovery = await crawler.arun("https://books.toscrape.com", config=prefetch_config)

discovery_time = time.time() - start

all_urls = [link["href"] for link in discovery.links.get("internal", [])]

print(f" Discovered {len(all_urls)} URLs in {discovery_time:.2f}s")

# Filter to URLs we care about (e.g., book detail pages)

# On books.toscrape.com, book pages contain "catalogue/" but not "category/"

book_urls = [

url for url in all_urls

if "catalogue/" in url and "category/" not in url

][:5] # Limit to 5 for demo

print(f" Filtered to {len(book_urls)} book pages")

# ═══════════════════════════════════════════════════════════

# Phase 2: Full processing on selected URLs

# ═══════════════════════════════════════════════════════════

print("\n--- Phase 2: Full Processing ---")

full_config = CrawlerRunConfig(

word_count_threshold=10,

remove_overlay_elements=True,

)

results = []

start = time.time()

for url in book_urls:

result = await crawler.arun(url, config=full_config)

if result.success:

results.append(result)

title = result.url.split("/")[-2].replace("-", " ").title()[:40]

md_len = len(result.markdown.raw_markdown) if result.markdown else 0

print(f" Processed: {title}... ({md_len} chars)")

processing_time = time.time() - start

print(f"\n Processed {len(results)} pages in {processing_time:.2f}s")

# ═══════════════════════════════════════════════════════════

# Summary

# ═══════════════════════════════════════════════════════════

print(f"\n--- Summary ---")

print(f" Discovery phase: {discovery_time:.2f}s ({len(all_urls)} URLs)")

print(f" Processing phase: {processing_time:.2f}s ({len(results)} pages)")

print(f" Total time: {discovery_time + processing_time:.2f}s")

print(f" URLs skipped: {len(all_urls) - len(book_urls)} (not matching filter)")

async def example_prefetch_with_deep_crawl():

"""

Example 4: Combine prefetch with deep crawl strategy.

Use prefetch mode during deep crawl for maximum speed.

"""

print("\n" + "=" * 60)

print("Example 4: Prefetch with Deep Crawl")

print("=" * 60)

from crawl4ai.deep_crawling import BFSDeepCrawlStrategy

async with AsyncWebCrawler(verbose=False) as crawler:

# Deep crawl with prefetch - maximum discovery speed

config = CrawlerRunConfig(

prefetch=True, # Fast mode

deep_crawl_strategy=BFSDeepCrawlStrategy(

max_depth=1,

max_pages=10,

)

)

print("\nDeep crawling with prefetch mode...")

start = time.time()

result_container = await crawler.arun("https://books.toscrape.com", config=config)

# Handle iterator result from deep crawl

if hasattr(result_container, '__iter__'):

results = list(result_container)

else:

results = [result_container]

elapsed = time.time() - start

# Collect all discovered links

all_internal_links = set()

all_external_links = set()

for result in results:

for link in result.links.get("internal", []):

all_internal_links.add(link["href"])

for link in result.links.get("external", []):

all_external_links.add(link["href"])

print(f"\nResults:")

print(f" Pages crawled: {len(results)}")

print(f" Total internal links discovered: {len(all_internal_links)}")

print(f" Total external links discovered: {len(all_external_links)}")

print(f" Time: {elapsed:.2f}s")

async def example_prefetch_with_raw_html():

"""

Example 5: Prefetch with raw HTML input.

You can also use prefetch mode with raw: URLs for cached content.

"""

print("\n" + "=" * 60)

print("Example 5: Prefetch with Raw HTML")

print("=" * 60)

sample_html = """

<html>

<head><title>Sample Page</title></head>

<body>

<h1>Hello World</h1>

<nav>

<a href="/page1">Internal Page 1</a>

<a href="/page2">Internal Page 2</a>

<a href="https://example.com/external">External Link</a>

</nav>

<main>

<p>This is the main content with <a href="/page3">another link</a>.</p>

</main>

</body>

</html>

"""

async with AsyncWebCrawler(verbose=False) as crawler:

config = CrawlerRunConfig(

prefetch=True,

base_url="https://mysite.com", # For resolving relative links

)

result = await crawler.arun(f"raw:{sample_html}", config=config)

print(f"\nExtracted from raw HTML:")

print(f" Internal links: {len(result.links.get('internal', []))}")

for link in result.links.get("internal", []):

print(f" - {link['href']} ({link['text']})")

print(f"\n External links: {len(result.links.get('external', []))}")

for link in result.links.get("external", []):

print(f" - {link['href']} ({link['text']})")

async def main():

"""Run all examples."""

print("=" * 60)

print("Prefetch Mode and Two-Phase Crawling Examples")

print("=" * 60)

await example_basic_prefetch()

await example_performance_comparison()

await example_two_phase_crawl()

await example_prefetch_with_deep_crawl()

await example_prefetch_with_raw_html()

if __name__ == "__main__":

asyncio.run(main())