Web Scraper Agents

pip3 install -U swarms swarms-tools

export OPENAI_API_KEY="..."

from swarms import Agent
from swarms_tools import scrape_and_format_sync

agent = Agent(
    agent_name="Web Scraper Agent",
    model_name="gpt-5.4",
    tools=[scrape_and_format_sync],
    dynamic_context_window=True,
    dynamic_temperature_enabled=True,
    max_loops=1,
    system_prompt=(
        "You are a web scraper agent. You are given a URL and you need to scrape "
        "the website and return the data in a structured format. The format type should be full"
    ),
)

out = agent.run(
    "Scrape swarms.ai website and provide a full report of the company does. "
    "The format type should be full."
)
print(out)

from swarms import Agent, batched_grid_agent_execution
from swarms_tools import scrape_and_format_sync


def make_scraper(name: str) -> Agent:
    return Agent(
        agent_name=name,
        model_name="gpt-5.4",
        tools=[scrape_and_format_sync],
        dynamic_context_window=True,
        dynamic_temperature_enabled=True,
        max_loops=1,
        system_prompt=(
            "You are a web scraper agent. You are given a URL and you need to scrape "
            "the website and return the data in a structured format. The format type should be full"
        ),
    )


out = batched_grid_agent_execution(
    agents=[make_scraper("Scraper-Swarms"), make_scraper("Scraper-LangChain")],
    tasks=[
        "Scrape swarms.ai website and provide a full report of the company's mission, "
        "products, and team. The format type should be full.",
        "Scrape langchain.com website and provide a full report of the company's mission, "
        "products, and team. The format type should be full.",
    ],
)

print(out)