Browser Automation with AI: Computer Vision & LLM

Imagine: a competitor's site updates its interface, and your RPA bot, tied to selectors like #price-block, starts throwing errors. Sound familiar? We solve this with AI agents that analyze the interface like a human — via screenshots and element semantics. They don't break on frontend refactoring and are resilient to anti-bot protection. Over 5+ years, we've delivered 30+ automation projects for e-commerce, logistics, and finance, saving clients an average of 70% manual work.

Traditional RPA bots are rigidly tied to DOM structure. Change a button's class — the script crashes. AI agents analyze screenshots and element semantics, so they work with any interface — SPA, React, Vue, dynamic elements. Flexibility is orders of magnitude higher: frontend refactoring doesn't break the agent.

What is an AI Agent and How is It Different from RPA?

An AI agent is a software bot that uses a multimodal large language model (LLM) and computer vision to interact with web interfaces. Unlike RPA, which follows rigid selectors, an AI agent sees the page like a human: it recognizes buttons, input fields, and other elements by their visual appearance and context. This makes it resilient to layout changes, framework swaps, and even some anti-bot protections.

How We Build the Agent: Architecture

We use a combination of Playwright + LLM (Claude, GPT‑4). Playwright controls the browser; the LLM makes decisions — selecting the next action based on the screenshot and available elements.

from playwright.async_api import async_playwright, Page from anthropic import Anthropic import asyncio import base64 import json client = Anthropic() class AIWebAgent: def __init__(self, headless: bool = True): self.headless = headless async def run(self, task: str, start_url: str) -> str: async with async_playwright() as p: browser = await p.chromium.launch(headless=self.headless) context = await browser.new_context( viewport={"width": 1280, "height": 800}, user_agent="Mozilla/5.0 (compatible; research-bot/1.0)" ) page = await context.new_page() await page.goto(start_url) result = await self._agent_loop(page, task) await browser.close() return result async def _get_page_state(self, page: Page) -> dict: screenshot = await page.screenshot(type="png") screenshot_b64 = base64.standard_b64encode(screenshot).decode() elements = await page.evaluate("""() => { const interactive = document.querySelectorAll( 'a, button, input, select, textarea, [role="button"]' ); return Array.from(interactive).slice(0, 50).map((el, idx) => ({ index: idx, tag: el.tagName.toLowerCase(), text: el.textContent?.trim().slice(0, 80) || '', type: el.type || '', placeholder: el.placeholder || '', })); }""") return { "url": page.url, "title": await page.title(), "screenshot": screenshot_b64, "elements": elements, } async def _execute_action(self, page: Page, action: dict) -> str: action_type = action.get("type") try: if action_type == "click": if "text" in action: await page.get_by_text(action["text"]).first.click() elif "element_index" in action: elements = await page.query_selector_all( 'a, button, input, select, textarea, [role="button"]' ) if action["element_index"] < len(elements): await elements[action["element_index"]].click() elif action_type == "fill": await page.fill(action["selector"], action["value"]) elif action_type == "navigate": await page.goto(action["url"]) elif action_type == "scroll": amount = action.get("amount", 500) direction = 1 if action.get("direction", "down") == "down" else -1 await page.evaluate(f"window.scrollBy(0, {direction * amount})") elif action_type == "extract": return await page.evaluate( f'document.querySelector({json.dumps(action.get("selector", "body"))})?.textContent?.trim()' ) await asyncio.sleep(0.5) return "success" except Exception as e: return f"error: {e}" async def _agent_loop(self, page: Page, task: str) -> str: messages = [] system = """You are an AI web agent. Complete tasks in the browser. Available actions (return JSON): - {"type": "click", "text": "button text"} - {"type": "fill", "selector": "CSS", "value": "text"} - {"type": "navigate", "url": "https://..."} - {"type": "scroll", "direction": "down", "amount": 500} - {"type": "extract", "selector": ".result"} - {"type": "done", "result": "final result"} Response format: REASONING: <what you see and plan> ACTION: <JSON with one action>""" for _ in range(20): state = await self._get_page_state(page) user_content = [ {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": state["screenshot"]}}, {"type": "text", "text": f"Task: {task}\nURL: {state['url']}\nElements:\n{json.dumps(state['elements'][:20], ensure_ascii=False)}"} ] messages.append({"role": "user", "content": user_content}) response = client.messages.create( model="claude-opus-4-5", max_tokens=1024, system=system, messages=messages, ) reply = response.content[0].text messages.append({"role": "assistant", "content": reply}) try: action_line = [l for l in reply.split("\n") if l.startswith("ACTION:")][0] action = json.loads(action_line.replace("ACTION:", "").strip()) except (IndexError, json.JSONDecodeError): continue if action.get("type") == "done": return action.get("result", "Task completed") await self._execute_action(page, action) return "Max iterations reached" 

async def setup_stealth_context(playwright): browser = await playwright.chromium.launch( headless=False, args=["--disable-blink-features=AutomationControlled"] ) context = await browser.new_context( viewport={"width": 1366, "height": 768}, locale="en-US", timezone_id="America/New_York", ) await context.add_init_script( "Object.defineProperty(navigator, 'webdriver', {get: () => undefined})" ) return context