OpenAI computer_use Tool Configuration (Display, Environment, Model)

from openai import OpenAI

client = OpenAI()


# ✅ PREVIEW shape — explicit dimensions and environment required
PREVIEW_TOOL = {
    "type": "computer_use_preview",
    "display_width": 1280,        # matches your headless browser viewport
    "display_height": 800,
    "environment": "browser",     # browser | windows | mac | ubuntu
}


# ✅ GA shape — simpler, defaults apply
GA_TOOL = {"type": "computer"}


# ✅ GA shape with explicit dimensions (still works, overrides defaults)
GA_TOOL_EXPLICIT = {
    "type": "computer",
    "display_width": 1920,
    "display_height": 1080,
    "environment": "browser",
}


# ✅ Initial call with screenshot in input
def start_computer_task(goal: str, initial_screenshot_b64: str, viewport=(1280, 800)):
    tool = {
        "type": "computer_use_preview",       # or "computer" for GA
        "display_width": viewport[0],
        "display_height": viewport[1],
        "environment": "browser",
    }

    resp = client.responses.create(
        model="gpt-5.4",                       # gpt-5.4 or later
        tools=[tool],
        input=[
            {
                "role": "user",
                "content": [
                    {"type": "input_text", "text": goal},
                    {
                        "type": "input_image",
                        "image_url": f"data:image/png;base64,{initial_screenshot_b64}",
                    },
                ],
            }
        ],
        truncation="auto",                     # trim old screenshots automatically
    )
    return resp


# ✅ Environment options and their calibration effects
ENVIRONMENT_HINTS = {
    "browser": {
        "typical_viewport": (1280, 800),
        "input_devices": "mouse + keyboard",
        "notes": "Chrome/Firefox/Edge in browser mode. Model assumes browser controls (URL bar, back button).",
    },
    "windows": {
        "typical_viewport": (1920, 1080),
        "input_devices": "mouse + keyboard",
        "notes": "Windows desktop apps. Model assumes Windows title bars, taskbar, start menu.",
    },
    "mac": {
        "typical_viewport": (1440, 900),
        "input_devices": "mouse + keyboard",
        "notes": "macOS. Cmd instead of Ctrl for shortcuts.",
    },
    "ubuntu": {
        "typical_viewport": (1920, 1080),
        "input_devices": "mouse + keyboard",
        "notes": "Linux desktop. Model assumes typical Ubuntu UI patterns.",
    },
}


# ✅ Common viewport matches
STANDARD_VIEWPORTS = {
    "browser_default":      (1280, 800),
    "browser_hd":           (1920, 1080),
    "browser_mobile":       (390, 844),        # iPhone 14 Pro
    "browser_tablet":       (1024, 768),        # iPad
    "windows_default":      (1920, 1080),
    "mac_retina":           (2560, 1600),
}


# ✅ Runtime-declared match — read actual browser dims and pass to tool
async def start_with_actual_dims(page, goal: str):
    """Playwright example — get actual viewport and pass to tool config."""
    viewport = page.viewport_size or {"width": 1280, "height": 800}

    screenshot = await page.screenshot()
    import base64
    screenshot_b64 = base64.b64encode(screenshot).decode()

    resp = client.responses.create(
        model="gpt-5.4",
        tools=[{
            "type": "computer_use_preview",
            "display_width": viewport["width"],
            "display_height": viewport["height"],
            "environment": "browser",
        }],
        input=[
            {
                "role": "user",
                "content": [
                    {"type": "input_text", "text": goal},
                    {"type": "input_image",
                     "image_url": f"data:image/png;base64,{screenshot_b64}"},
                ],
            }
        ],
        truncation="auto",
    )
    return resp


# ❌ ANTI-PATTERN — declared vs actual dimension mismatch
# Playwright default is 1280×720, but you declared 1920×1080 to the tool.
# Model returns clicks in 1920×1080 space; they land off-target in the actual page.
# Fix: always read the ACTUAL viewport and pass those dims to the tool.


# ❌ ANTI-PATTERN — GA "computer" shape with old model
# resp = client.responses.create(
#     model="gpt-4.1",                    # too old
#     tools=[{"type": "computer"}],
#     ...
# )
# BadRequestError: Tool 'computer' is not supported for model 'gpt-4.1'


# ❌ ANTI-PATTERN — preview shape without dimensions
# resp = client.responses.create(
#     model="gpt-5.4",
#     tools=[{"type": "computer_use_preview"}],   # missing display_width/height/environment
#     ...
# )
# BadRequestError: Field 'display_width' is required

from playwright.async_api import async_playwright
import base64


# ✅ APPROACH 1 — force browser to declared dims
async def launch_at_declared_dims(width: int = 1280, height: int = 800):
    """Playwright launched with a specific viewport that matches declared dims."""
    p = await async_playwright().start()
    browser = await p.chromium.launch(headless=True)
    context = await browser.new_context(viewport={"width": width, "height": height})
    page = await context.new_page()
    return p, browser, page


# ✅ APPROACH 2 — read actual dims, pass to tool
async def declare_from_actual(page):
    """Read Playwright's actual viewport, use those for tool declaration."""
    viewport = page.viewport_size or {"width": 1280, "height": 800}
    return {
        "type": "computer_use_preview",
        "display_width": viewport["width"],
        "display_height": viewport["height"],
        "environment": "browser",
    }


# ✅ APPROACH 3 — scale coordinates on execution
class CoordinateScaler:
    """When declared and actual differ (e.g. Retina display), scale actions."""

    def __init__(self, declared: tuple[int, int], actual: tuple[int, int]):
        self.dx = actual[0] / declared[0]
        self.dy = actual[1] / declared[1]

    def scale_action(self, action: dict) -> dict:
        """Rewrite coordinates in a computer_use action to actual dims."""
        scaled = dict(action)
        if "x" in scaled:
            scaled["x"] = int(scaled["x"] * self.dx)
        if "y" in scaled:
            scaled["y"] = int(scaled["y"] * self.dy)

        # Drag actions have path arrays
        if "path" in scaled:
            scaled["path"] = [{"x": int(pt["x"] * self.dx), "y": int(pt["y"] * self.dy)}
                              for pt in scaled["path"]]

        # Scroll deltas can also be dimension-relative
        if "scroll_x" in scaled:
            scaled["scroll_x"] = int(scaled["scroll_x"] * self.dx)
        if "scroll_y" in scaled:
            scaled["scroll_y"] = int(scaled["scroll_y"] * self.dy)

        return scaled


# Example: model reasons in 1280×800 but our screen is 1920×1080
scaler = CoordinateScaler(declared=(1280, 800), actual=(1920, 1080))
scaled = scaler.scale_action({"type": "click", "x": 640, "y": 400, "button": "left"})
# scaled = {"type": "click", "x": 960, "y": 540, "button": "left"}


# ✅ Take screenshot at declared dims (downscale)
async def screenshot_at_declared(page, declared_w: int, declared_h: int) -> bytes:
    """Screenshot the current page, resize to declared dims before sending."""
    raw_bytes = await page.screenshot()
    from PIL import Image
    import io as _io

    img = Image.open(_io.BytesIO(raw_bytes))
    if img.size != (declared_w, declared_h):
        img = img.resize((declared_w, declared_h), Image.LANCZOS)

    out = _io.BytesIO()
    img.save(out, format="PNG")
    return out.getvalue()


# ✅ Mobile viewport support
async def mobile_computer_use():
    """iPhone-sized viewport for testing mobile web flows."""
    p = await async_playwright().start()
    iphone = p.devices["iPhone 14 Pro"]
    browser = await p.chromium.launch(headless=True)
    context = await browser.new_context(**iphone)
    page = await context.new_page()
    await page.goto("https://example.com")

    tool = {
        "type": "computer_use_preview",
        "display_width": iphone["viewport"]["width"],
        "display_height": iphone["viewport"]["height"],
        "environment": "browser",
    }
    return page, tool


# ✅ Verification pattern — after every N actions, screenshot + compare
async def verify_declared_match(page, declared_w: int, declared_h: int):
    """Sanity check: is Playwright still rendering at declared dims?"""
    actual = page.viewport_size
    if not actual or actual["width"] != declared_w or actual["height"] != declared_h:
        raise RuntimeError(
            f"viewport drift: declared={declared_w}×{declared_h}, "
            f"actual={actual['width'] if actual else '?'}×{actual['height'] if actual else '?'}"
        )

import base64
from openai import OpenAI

client = OpenAI()


# ❌ ANTI-PATTERN — vague goal, no screenshot
def bad_start():
    return client.responses.create(
        model="gpt-5.4",
        tools=[{"type": "computer_use_preview", "display_width": 1280,
                "display_height": 800, "environment": "browser"}],
        input="Do the thing on the page.",     # too vague
    )


# ✅ GOOD — screenshot + explicit goal + constraints
def good_start(screenshot_b64: str):
    return client.responses.create(
        model="gpt-5.4",
        tools=[{"type": "computer_use_preview", "display_width": 1280,
                "display_height": 800, "environment": "browser"}],
        instructions=(
            "You are a browser automation agent. "
            "Complete the task by clicking, typing, and scrolling as needed. "
            "IMPORTANT: Never enter passwords, credit card numbers, or SSNs "
            "without explicit permission. If you encounter a login screen "
            "you weren't authorized to complete, stop and report back."
        ),
        input=[
            {
                "role": "user",
                "content": [
                    {
                        "type": "input_text",
                        "text": (
                            "Task: Find the pricing page and report back the price "
                            "of the \"Pro\" plan (monthly billing). Do not click any "
                            "\"Buy\" or \"Subscribe\" buttons — just read the price."
                        ),
                    },
                    {
                        "type": "input_image",
                        "image_url": f"data:image/png;base64,{screenshot_b64}",
                    },
                ],
            }
        ],
        truncation="auto",
    )


# ✅ Structured task template
def make_task_input(
    goal: str,
    constraints: list[str],
    success_criteria: str,
    screenshot_b64: str,
    starting_url: str | None = None,
) -> list:
    task_lines = [f"TASK: {goal}"]

    if starting_url:
        task_lines.append(f"STARTING URL: {starting_url}")

    if constraints:
        task_lines.append("CONSTRAINTS:")
        task_lines.extend(f"  - {c}" for c in constraints)

    task_lines.append(f"SUCCESS: {success_criteria}")
    task_lines.append("")
    task_lines.append("Report back when you have the answer, or if you get stuck.")

    return [
        {
            "role": "user",
            "content": [
                {"type": "input_text", "text": "\n".join(task_lines)},
                {"type": "input_image", "image_url": f"data:image/png;base64,{screenshot_b64}"},
            ],
        }
    ]


# ✅ Example use
input_data = make_task_input(
    goal="Extract the top 5 news headlines from the front page",
    constraints=[
        "Don't click on individual articles",
        "Don't click on ads or paywall prompts",
        "Report headlines exactly as displayed",
    ],
    success_criteria="Return a numbered list of 5 headlines and their URLs",
    screenshot_b64="...",
    starting_url="https://example-news-site.com",
)


# ✅ For agents that need to run multiple similar tasks, template the instructions
INSTRUCTIONS_TEMPLATE = """You are a browser automation agent operating in a {environment} environment
at {width}×{height} pixels.

You can:
- click(x, y): click at coordinates
- type(text): type text into focused input
- key(name): press a key (Enter, Tab, Escape, ArrowDown, etc.)
- scroll(x, y, delta_x, delta_y): scroll in direction
- wait(): wait for page to settle
- screenshot(): take a fresh screenshot

Guidelines:
- Take a screenshot after any action that changes page state
- If the page is loading, use wait() before continuing
- Report progress every 3-5 actions
- If the goal requires payment, personal data, or destructive action, STOP and ask

Safety:
- Never enter credentials
- Never accept unknown terms/conditions
- If uncertain, take a screenshot and describe what you see"""


def make_instructions(environment: str, width: int, height: int) -> str:
    return INSTRUCTIONS_TEMPLATE.format(environment=environment, width=width, height=height)