Scrapit Documentation

$ pip install scrapit-scraper

# Playwright backend (JS-rendered sites)
$ pip install scrapit-scraper[playwright]
$ playwright install chromium

# AI integrations
$ pip install scrapit-scraper[anthropic]   # Claude
$ pip install scrapit-scraper[openai]      # GPT-4o
$ pip install scrapit-scraper[langchain]   # LangChain / CrewAI
$ pip install scrapit-scraper[llamaindex]  # LlamaIndex
$ pip install scrapit-scraper[mcp]         # MCP server

# Storage backends
$ pip install scrapit-scraper[mongo]
$ pip install scrapit-scraper[excel]
$ pip install scrapit-scraper[sheets]

# Everything
$ pip install scrapit-scraper[all]

$ git clone https://github.com/joaobenedetmachado/scrapit
$ cd scrapit
$ python -m venv .venv && source .venv/bin/activate
$ pip install -e .[dev]

site: https://news.ycombinator.com
use: beautifulsoup

scrape:
  titles:
    - '.titleline > a'
    - attr: text
      all: true
  links:
    - '.titleline > a'
    - attr: href
      all: true

$ scrapit scrape hn --json
→ saved: output/hn.json

$ scrapit scrape hn --json --diff  # detect changes
$ scrapit scrape hn --csv --sqlite  # multiple outputs
$ scrapit scrape hn --preview       # print, don't save

$ scrapit query --backend sqlite --directive hn --limit 5

$ scrapit batch scraper/directives/ --json --sqlite

$ scrapit list
$ scrapit list --dir /path/to/my/directives

$ scrapit cache stats
$ scrapit cache clear
$ scrapit cache invalidate --url https://example.com

# MongoDB
MONGO_URI=mongodb://localhost:27017
MONGO_DB=scrapit
MONGO_COLLECTION=results

# RabbitMQ
RABBITMQ_HOST=localhost
RABBITMQ_QUEUE=scrapit

# Output directory (default: ./output)
OUTPUT_DIR=./output

scrape:
  # Single selector, get text
  title:
    - 'h1'
    - attr: text

  # Get an HTML attribute
  image:
    - 'img.hero'
    - attr: src

  # Return ALL matches as a list
  all_links:
    - 'a.result'
    - attr: href
      all: true

  # Fallback selectors — tries each in order
  price:
    - ['span.price-new', 'span.price', '.cost']
    - attr: text

paginate:
  selector: 'a.next'     # CSS selector for the next-page link
  attr: href              # attribute to extract the URL from
  max_pages: 5           # safety limit

site: https://myblog.com/posts
use: beautifulsoup

follow:
  selector: 'a.post-link'
  attr: href
  max: 100
  same_domain: true

scrape:
  title:
    - 'h1'
    - attr: text
  body:
    - 'article'
    - attr: text

headers:
  Accept-Language: en-US,en;q=0.9
  Referer: https://google.com

cookies:
  session_id: abc123

proxy: http://user:pass@proxy:8080

cache:
  ttl: 3600    # seconds. 0 to disable.

retries: 3    # exponential backoff

# Playwright only
wait_for: '#content'
screenshot: true

notify:
  webhook: https://hooks.slack.com/services/...

sites:
  - https://books.toscrape.com/catalogue/page-1.html
  - https://books.toscrape.com/catalogue/page-2.html
  - https://books.toscrape.com/catalogue/page-3.html

use: beautifulsoup

scrape:
  titles:
    - 'h3 > a'
    - attr: title
      all: true

transform:
  price:
    - strip                      # "  £ 12,99  " → "£ 12,99"
    - replace: {"£": "", ",": "."}
    - float                      # → 12.99

  slug:
    - strip
    - normalize_whitespace
    - slugify                    # "My  Title!" → "my-title"

  summary:
    - remove_tags
    - normalize_whitespace
    - truncate: 200

validate:
  price:
    required: true
    type: float
    min: 0
  title:
    required: true
    min_length: 2
    max_length: 500
  status:
    in: [active, inactive, pending]
  sku:
    pattern: '^[A-Z]{2}\d{4}$'

from scraper import hooks

@hooks.on("after_scrape")
def log_result(result, dados):
    print(f"scraped {result['url']} — {len(result)} fields")

@hooks.on("on_change")
def alert(changes, result):
    print(f"changed fields: {list(changes.keys())}")

@hooks.on("on_error")
def handle_error(exc, dados):
    print(f"failed on {dados['site']}: {exc}")

$ pip install scrapit-scraper[mongo]

MONGO_URI=mongodb://localhost:27017
MONGO_DB=scrapit
MONGO_COLLECTION=results

from scraper.integrations import (
    scrape_url,             # clean text
    scrape_page,            # structured metadata
    scrape_with_selectors,  # CSS-driven extraction
    scrape_many,            # parallel URLs
    scrape_directive,       # run a YAML directive
)

text = scrape_url("https://example.com")

page = scrape_page("https://example.com")
# → {url, title, description, main_content, links, word_count}

data = scrape_with_selectors(
    "https://books.toscrape.com/...",
    selectors={"title": "h1", "price": "p.price_color"},
    all_matches={"tags": True},
)

pages = scrape_many(
    ["https://a.com", "https://b.com"],
    mode="page",
    max_workers=8,
)

$ pip install scrapit-scraper[mcp]

$ claude mcp add scrapit -- python -m scraper.integrations.mcp

{
  "mcpServers": {
    "scrapit": {
      "command": "python",
      "args": ["-m", "scraper.integrations.mcp"],
      "cwd": "/path/to/scrapit"
    }
  }
}

$ pip install scrapit-scraper[anthropic]

from scraper.integrations.anthropic import ScrapitAnthropicAgent

agent = ScrapitAnthropicAgent(
    model="claude-opus-4-6",
    max_iterations=10,
    system="You are a research assistant...",
)

answer = agent.run("What are the top 5 stories on Hacker News?")
print(answer)

import anthropic
from scraper.integrations.anthropic import as_anthropic_tools, handle_tool_call

client   = anthropic.Anthropic()
tools    = as_anthropic_tools()

response = client.messages.create(
    model="claude-opus-4-6", max_tokens=1024,
    tools=tools,
    messages=[{"role": "user", "content": "..."}],
)

for block in response.content:
    if block.type == "tool_use":
        result = handle_tool_call(block.name, block.input)

$ pip install scrapit-scraper[openai]

from scraper.integrations.openai import ScrapitOpenAIAgent

agent  = ScrapitOpenAIAgent(model="gpt-4o")
answer = agent.run("Summarize the Python Wikipedia page.")
print(answer)

from openai import OpenAI
from scraper.integrations.openai import as_openai_functions, handle_function_call

client   = OpenAI()
tools    = as_openai_functions()

response = client.chat.completions.create(
    model="gpt-4o", tools=tools,
    messages=[{"role": "user", "content": "..."}],
)

for call in response.choices[0].message.tool_calls:
    result = handle_function_call(call.function.name, call.function.arguments)

$ pip install scrapit-scraper[langchain]

from scraper.integrations.langchain import ScrapitToolkit

tools = ScrapitToolkit().get_tools()
tools = ScrapitToolkit(directives=["wikipedia", "hn"]).get_tools()

from langchain.agents import initialize_agent, AgentType
from langchain_openai import ChatOpenAI

agent = initialize_agent(
    tools=ScrapitToolkit().get_tools(),
    llm=ChatOpenAI(model="gpt-4o"),
    agent=AgentType.OPENAI_FUNCTIONS,
)
agent.run("What's on the front page of Hacker News?")

from crewai import Agent

researcher = Agent(
    role="Web Researcher",
    goal="Find and summarize information from the web",
    tools=ScrapitToolkit().get_tools(),
    llm="gpt-4o",
)

from scraper.integrations.langchain import ScrapitLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter

loader = ScrapitLoader("https://example.com")
docs   = loader.load()
chunks = RecursiveCharacterTextSplitter(chunk_size=1000).split_documents(docs)

$ pip install scrapit-scraper[llamaindex]

from scraper.integrations.llamaindex import ScrapitReader
from llama_index.core import VectorStoreIndex

reader = ScrapitReader()

docs = reader.load_data(url="https://example.com")
docs = reader.load_data(urls=["https://a.com", "https://b.com"])
docs = reader.load_data(directive="wikipedia")

engine = VectorStoreIndex.from_documents(docs).as_query_engine()
result = engine.query("What is the main topic?")

import asyncio
from scraper.scrapers import grab_elements_by_directive
from scraper.storage import json_file, sqlite

result = asyncio.run(
    grab_elements_by_directive("scraper/directives/hn.yaml")
)

json_file.save(result, "hn")
sqlite.save(result, "hn")

$ pip install scrapit-scraper[rabbitmq]

# Producer — send a directive to the queue
from scraper.queue.producer import call_producer
call_producer("directives/hn.yaml")

# Consumer — start a blocking worker
$ python -m scraper.queue.consumer