Scrape any site
with a YAML file. // no code needed

# Claude Code
$ claude mcp add scrapit -- \
    python -m scraper.integrations.mcp

# claude_desktop_config.json
{
  "mcpServers": {
    "scrapit": {
      "command": "python",
      "args": ["-m", "scraper.integrations.mcp"]
    }
  }
}

from scraper.integrations.anthropic import (
    ScrapitAnthropicAgent,
    as_anthropic_tools,
    handle_tool_call,
)

# Built-in agentic loop
agent = ScrapitAnthropicAgent(
    model="claude-opus-4-6"
)
answer = agent.run(
    "What are today's top HN posts?"
)

from scraper.integrations.openai import (
    ScrapitOpenAIAgent,
    as_openai_functions,
    handle_function_call,
)

# Built-in agentic loop
agent = ScrapitOpenAIAgent(model="gpt-4o")
answer = agent.run(
    "Summarize the Python Wikipedia page."
)

from scraper.integrations.langchain import ScrapitToolkit

tools = ScrapitToolkit().get_tools()
# → [ScrapitTool, ScrapitPageTool, ScrapitSelectorTool]

# LangChain agent
agent = initialize_agent(
    tools=tools, llm=ChatOpenAI(model="gpt-4o"),
    agent=AgentType.OPENAI_FUNCTIONS,
)

# CrewAI — same tools, different framework
researcher = Agent(role="Researcher", tools=tools)

from scraper.integrations.llamaindex import ScrapitReader
from llama_index.core import VectorStoreIndex

reader = ScrapitReader()

# Parallel URL scraping
docs = reader.load_data(urls=[
    "https://site1.com",
    "https://site2.com",
])

index  = VectorStoreIndex.from_documents(docs)
engine = index.as_query_engine()
engine.query("Summarize the main points.")

from scraper.integrations import (
    scrape_url,           # → clean text
    scrape_page,          # → structured metadata
    scrape_with_selectors,# → CSS-defined fields
    scrape_many,          # → parallel URLs
)

page = scrape_page("https://example.com")
# → {title, description, main_content, links, word_count}

data = scrape_with_selectors("https://example.com", {
    "title": "h1",
    "price": ".price-color",
})