GeezerVison

pip install "scrapegraph-py>=2.1.0"
# or
uv add "scrapegraph-py>=2.1.0"

from scrapegraph_py import ScrapeGraphAI

# reads SGAI_API_KEY from env, or pass it explicitly:
# sgai = ScrapeGraphAI(api_key="sgai-...")
sgai = ScrapeGraphAI()

result = sgai.scrape("https://example.com")

if result.status == "success":
    print(result.data.results["markdown"]["data"])
else:
    print(result.error)

from typing import Generic, Literal, TypeVar
from pydantic import BaseModel

T = TypeVar("T")

class ApiResult(BaseModel, Generic[T]):
    status: Literal["success", "error"]
    data: T | None
    error: str | None = None
    elapsed_ms: int

with ScrapeGraphAI() as sgai:
    result = sgai.scrape("https://example.com")

from scrapegraph_py import (
    ScrapeGraphAI, FetchConfig,
    MarkdownFormatConfig, ScreenshotFormatConfig, JsonFormatConfig,
)

sgai = ScrapeGraphAI()

res = sgai.scrape(
    "https://example.com",
    formats=[
        MarkdownFormatConfig(mode="reader"),
        ScreenshotFormatConfig(full_page=True, width=1440, height=900),
        JsonFormatConfig(prompt="Extract product info"),
    ],
    content_type="text/html",  # optional, auto-detected
    fetch_config=FetchConfig(
        mode="js",
        stealth=True,
        timeout=30000,
        wait=2000,
        scrolls=3,
    ),
)

if res.status == "success":
    markdown = res.data.results["markdown"]["data"]

from scrapegraph_py import ScrapeGraphAI

sgai = ScrapeGraphAI()

res = sgai.extract(
    "Extract product names and prices",
    url="https://example.com",
    schema={
        "type": "object",
        "properties": {
            "products": {
                "type": "array",
                "items": {
                    "type": "object",
                    "properties": {
                        "name":  {"type": "string"},
                        "price": {"type": "string"},
                    },
                },
            },
        },
    },
)

if res.status == "success":
    print(res.data.json_data)
    print(f"Tokens: {res.data.usage.prompt_tokens} / {res.data.usage.completion_tokens}")

from pydantic import BaseModel, Field
from scrapegraph_py import ScrapeGraphAI

class Product(BaseModel):
    name: str
    price: str | None = None

class Products(BaseModel):
    products: list[Product] = Field(default_factory=list)

sgai = ScrapeGraphAI()

res = sgai.extract(
    "Extract product names and prices",
    url="https://example.com",
    schema=Products.model_json_schema(),
)

if res.status == "success":
    parsed = Products.model_validate(res.data.json_data)
    for p in parsed.products:
        print(p.name, p.price)

from scrapegraph_py import ScrapeGraphAI

sgai = ScrapeGraphAI()

res = sgai.search(
    "best programming languages 2024",
    num_results=5,
    prompt="Summarize the top languages and reasons",
    time_range="past_week",
    location_geo_code="us",
)

if res.status == "success":
    for hit in res.data.results:
        print(hit.title, hit.url)
    print(res.data.json_data)  # when prompt/schema are set

from scrapegraph_py import ScrapeGraphAI, MarkdownFormatConfig

sgai = ScrapeGraphAI()

# Start
start = sgai.crawl.start(
    "https://example.com",
    formats=[MarkdownFormatConfig()],
    max_depth=2,
    max_pages=50,
    max_links_per_page=10,
    include_patterns=["/blog/*"],
    exclude_patterns=["/admin/*"],
)

crawl_id = start.data.id

# Poll
status = sgai.crawl.get(crawl_id)
print(f"{status.data.finished}/{status.data.total} - {status.data.status}")

# Control
sgai.crawl.stop(crawl_id)
sgai.crawl.resume(crawl_id)
sgai.crawl.delete(crawl_id)

from scrapegraph_py import ScrapeGraphAI, MarkdownFormatConfig

sgai = ScrapeGraphAI()

mon = sgai.monitor.create(
    "https://example.com",
    "0 * * * *",                 # cron expression (positional)
    name="Price Monitor",
    formats=[MarkdownFormatConfig()],
    webhook_url="https://example.com/webhook",
)

cron_id = mon.data.cron_id

sgai.monitor.list()
sgai.monitor.get(cron_id)
sgai.monitor.update(cron_id, interval="0 */6 * * *")
sgai.monitor.pause(cron_id)
sgai.monitor.resume(cron_id)
sgai.monitor.delete(cron_id)

act = sgai.monitor.activity(cron_id, limit=20)

if act.status == "success":
    for tick in act.data.ticks:
        status = "CHANGED" if tick.changed else "no change"
        print(f"[{tick.created_at}] {tick.status} - {status} ({tick.elapsed_ms}ms)")

    if act.data.next_cursor:
        more = sgai.monitor.activity(cron_id, limit=20, cursor=act.data.next_cursor)

from scrapegraph_py import ScrapeGraphAI

sgai = ScrapeGraphAI()

page = sgai.history.list(service="scrape", page=1, limit=20)
for entry in page.data.data:
    print(entry.id, entry.service, entry.status, entry.elapsed_ms)

one = sgai.history.get("request-id")

credits = sgai.credits()
# ApiResult[CreditsResponse] with .remaining, .used, .plan, .jobs.crawl, .jobs.monitor

health = sgai.health()
# ApiResult[HealthResponse] with .status, .uptime, .services

from scrapegraph_py import FetchConfig

config = FetchConfig(
    mode="js",            # "auto" (default), "fast", "js"
    stealth=True,         # Residential proxies / anti-bot headers (+5 credits)
    timeout=30000,        # 1,000–60,000 ms
    wait=2000,            # 0–30,000 ms
    scrolls=3,            # 0–100
    country="us",         # ISO 3166-1 alpha-2
    headers={"X-Custom": "header"},
    cookies={"session": "abc"},
    mock=False,           # Or a MockConfig object for testing
)

import asyncio
from scrapegraph_py import AsyncScrapeGraphAI

async def main():
    async with AsyncScrapeGraphAI() as sgai:
        res = await sgai.scrape("https://example.com")
        if res.status == "success":
            print(res.data.results["markdown"]["data"])

        start = await sgai.crawl.start("https://example.com", max_pages=25)
        status = await sgai.crawl.get(start.data.id)
        print(status.data.status)

        credits = await sgai.credits()
        print(credits.data.remaining)

asyncio.run(main())