Python Web Scraping — Beautiful Soup, Scrapy, httpx, and Playwright

import httpx

# Basic GET
response = httpx.get("https://httpbin.org/json")
print(response.status_code)  # 200
data = response.json()
print(data["slideshow"]["title"])

# With query parameters
params = {"q": "web scraping", "page": 2}
response = httpx.get("https://httpbin.org/get", params=params)
print(response.url)  # Includes ?q=...&page=2

# Custom headers
headers = {
    "User-Agent": "Mozilla/5.0 (compatible; DodaTechBot/1.0)",
    "Accept": "text/html,application/xhtml+xml",
}
response = httpx.get("https://example.com", headers=headers)

# POST with JSON body
response = httpx.post(
    "https://httpbin.org/post",
    json={"query": "search term"}
)

# Timeout and retries
transport = httpx.HTTPTransport(retries=3)
client = httpx.Client(transport=transport, timeout=30.0)

# Async client
async with httpx.AsyncClient() as client:
    response = await client.get("https://example.com")

from bs4 import BeautifulSoup
import httpx

response = httpx.get("https://example.com")
soup = BeautifulSoup(response.text, "html.parser")

# Find elements
title = soup.title.text
first_paragraph = soup.find("p").text
all_links = soup.find_all("a")

# CSS selectors
main_content = soup.select_one("div.main-content")
nav_items = soup.select("nav ul li a")

# Extract attributes
for link in soup.select("a[href]"):
    url = link["href"]
    text = link.get_text(strip=True)
    print(f"{text}: {url}")

# Navigating the tree
for child in soup.body.children:
    if child.name == "div":
        print(f"Found div: {child.get('class')}")

# Find by text
results = soup.find_all(string=lambda text: "Python" in text)

# pip install scrapyd
# spiders/quotes_spider.py
import scrapy

class QuotesSpider(scrapy.Spider):
    name = "quotes"
    start_urls = ["https://quotes.toscrape.com/page/1/"]

    def parse(self, response):
        for quote in response.css("div.quote"):
            yield {
                "text": quote.css("span.text::text").get(),
                "author": quote.css("small.author::text").get(),
                "tags": quote.css("div.tags a.tag::text").getall(),
            }

        # Follow pagination
        next_page = response.css("li.next a::attr(href)").get()
        if next_page:
            yield response.follow(next_page, self.parse)

# items.py
import scrapy

class ScrapedItem(scrapy.Item):
    url = scrapy.Field()
    title = scrapy.Field()
    content = scrapy.Field()
    scrape_date = scrapy.Field()

# pipelines.py
import json
from datetime import datetime

class JsonPipeline:
    def open_spider(self, spider):
        self.file = open("output.jsonl", "w")

    def close_spider(self, spider):
        self.file.close()

    def process_item(self, item, spider):
        item["scrape_date"] = datetime.now().isoformat()
        self.file.write(json.dumps(dict(item)) + "\n")
        return item

from playwright.sync_api import sync_playwright
import time

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    page = browser.new_page()

    # Navigate and wait for content to render
    page.goto("https://example.com")
    page.wait_for_load_state("networkidle")

    # Wait for specific element
    page.wait_for_selector("div.dynamic-content", timeout=5000)

    # Extract rendered HTML
    content = page.content()
    title = page.title()

    # Interact with page
    page.fill("input#search", "Python")
    page.click("button[type=submit]")
    page.wait_for_load_state("networkidle")

    # Extract results
    results = page.query_selector_all("div.result")
    for result in results:
        print(result.inner_text())

    # Screenshot for debugging
    page.screenshot(path="debug.png")

    browser.close()

from playwright.async_api import async_playwright
import asyncio

async def scrape_js_pages(urls):
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        context = await browser.new_context(
            user_agent="Mozilla/5.0 (compatible; DodaTechBot/1.0)"
        )

        results = []
        for url in urls:
            page = await context.new_page()
            await page.goto(url, wait_until="networkidle")
            data = await page.evaluate("""
                () => ({
                    title: document.title,
                    h1s: Array.from(document.querySelectorAll('h1')).map(h => h.textContent),
                    links: Array.from(document.querySelectorAll('a[href]')).map(a => a.href)
                })
            """)
            results.append(data)
            await page.close()

        await browser.close()
        return results

import asyncio
import time
from collections import deque

class PoliteScraper:
    def __init__(self, delay=2.0, max_concurrent=5):
        self.delay = delay
        self.semaphore = asyncio.Semaphore(max_concurrent)
        self.last_request = {}

    async def fetch(self, url, domain):
        async with self.semaphore:
            # Ensure minimum delay between requests to same domain
            now = time.time()
            last = self.last_request.get(domain, 0)
            wait = self.delay - (now - last)
            if wait > 0:
                await asyncio.sleep(wait)

            self.last_request[domain] = time.time()

            async with httpx.AsyncClient() as client:
                response = await client.get(
                    url,
                    headers={"User-Agent": "PoliteBot/1.0"}
                )
                response.raise_for_status()
                return response

    async def scrape_pages(self, urls):
        results = []
        for url in urls:
            from urllib.parse import urlparse
            domain = urlparse(url).netloc
            result = await self.fetch(url, domain)
            results.append(result)
        return results

from urllib.robotparser import RobotFileParser
import httpx

def check_allowed(url, user_agent="PoliteBot/1.0"):
    from urllib.parse import urlparse
    parsed = urlparse(url)
    robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt"

    rp = RobotFileParser()
    rp.set_url(robots_url)
    rp.read()

    return rp.can_fetch(user_agent, url)

# Test
url = "https://example.com/some-page"
if check_allowed(url):
    print("Allowed to scrape")
else:
    print("Blocked by robots.txt")