Claude Computer Use — tool_result malformed or action loop stuck

import anthropic

client = anthropic.Anthropic(
    default_headers={"anthropic-beta": "computer-use-2025-01-24"}  # verify current beta
)

# The tool declaration tells Claude the coordinate space it operates in
COMPUTER_TOOL = {
    "type": "computer_20250124",   # match the beta header version
    "name": "computer",
    "display_width_px": 1280,      # MUST match your actual screen/VM
    "display_height_px": 800,
    "display_number": 1,           # X display number (Linux VMs)
}

# Send the first message — Claude will respond with a tool_use (usually screenshot first)
response = client.messages.create(
    model="claude-opus-4-7",
    max_tokens=1024,
    tools=[COMPUTER_TOOL],
    messages=[{
        "role": "user",
        "content": "Open the file explorer and navigate to /home/user/reports.",
    }],
)

# Inspect what the model asks for
for block in response.content:
    if block.type == "tool_use":
        print(f"Model requests: {block.name}.{block.input.get('action')} @ {block.input}")

import base64
import hashlib
import subprocess
import anthropic
from io import BytesIO
from PIL import ImageGrab

client = anthropic.Anthropic(
    default_headers={"anthropic-beta": "computer-use-2025-01-24"}
)

MAX_ITERATIONS = 30
STUCK_THRESHOLD = 3  # same action + same screenshot 3 iterations = give up

def screenshot_b64() -> tuple[str, str]:
    """Grab screen, return (base64_png, sha256_hash)."""
    img = ImageGrab.grab()
    buf = BytesIO()
    img.save(buf, format="PNG")
    data = buf.getvalue()
    return base64.b64encode(data).decode(), hashlib.sha256(data).hexdigest()

def execute_action(action: dict) -> None:
    """Translate Claude's action to a local OS operation (Linux + xdotool)."""
    kind = action["action"]
    if kind == "screenshot":
        return  # nothing to do — screenshot is captured after
    elif kind in ("left_click", "right_click", "middle_click", "double_click"):
        x, y = action["coordinate"]
        button = {"left_click": "1", "right_click": "3", "middle_click": "2"}.get(kind, "1")
        clicks = "2" if kind == "double_click" else "1"
        subprocess.run(["xdotool", "mousemove", str(x), str(y),
                       "click", "--repeat", clicks, button], check=True)
    elif kind == "type":
        subprocess.run(["xdotool", "type", "--", action["text"]], check=True)
    elif kind == "key":
        subprocess.run(["xdotool", "key", action["text"]], check=True)
    # ... other actions omitted for brevity

def run_agent(task: str) -> anthropic.types.Message:
    messages = [{"role": "user", "content": task}]

    prev_action_key = None
    prev_screenshot_hash = None
    stuck_count = 0

    for iteration in range(MAX_ITERATIONS):
        response = client.messages.create(
            model="claude-opus-4-7",
            max_tokens=1024,
            tools=[{
                "type": "computer_20250124", "name": "computer",
                "display_width_px": 1280, "display_height_px": 800,
            }],
            messages=messages,
        )
        messages.append({"role": "assistant", "content": response.content})

        if response.stop_reason != "tool_use":
            return response  # Claude is done

        # Execute all tool_use blocks in the response
        tool_results = []
        for block in response.content:
            if block.type != "tool_use":
                continue
            execute_action(block.input)
            b64, screenshot_hash = screenshot_b64()

            # Stuck detection
            action_key = f"{block.input.get('action')}:{block.input.get('coordinate')}"
            if action_key == prev_action_key and screenshot_hash == prev_screenshot_hash:
                stuck_count += 1
                if stuck_count >= STUCK_THRESHOLD:
                    raise RuntimeError(f"Stuck at iteration {iteration}: {action_key}")
            else:
                stuck_count = 0
            prev_action_key = action_key
            prev_screenshot_hash = screenshot_hash

            tool_results.append({
                "type": "tool_result",
                "tool_use_id": block.id,
                "content": [{
                    "type": "image",
                    "source": {"type": "base64", "media_type": "image/png", "data": b64},
                }],
            })

        messages.append({"role": "user", "content": tool_results})

    raise RuntimeError(f"Iteration cap ({MAX_ITERATIONS}) reached without completion.")

result = run_agent("Open the terminal and run: ls -la /home")
print(result.content[-1].text)

from PIL import Image, ImageGrab
from io import BytesIO
import base64

TARGET_MAX_DIM = 1280  # matches your declared display width

def screenshot_scaled_b64() -> tuple[str, tuple[int, int]]:
    """Grab, downscale to TARGET_MAX_DIM, return (base64, new_dims).

    IMPORTANT: When you downscale, the coordinates Claude returns are in the
    DOWNSCALED space. You must upscale them back before executing clicks on the
    real display. Alternatively, declare the tool with the downscaled dims.
    """
    img = ImageGrab.grab()
    w, h = img.size
    scale = min(TARGET_MAX_DIM / max(w, h), 1.0)
    if scale < 1.0:
        img = img.resize((int(w * scale), int(h * scale)), Image.LANCZOS)

    buf = BytesIO()
    img.save(buf, format="PNG", optimize=True)
    return base64.b64encode(buf.getvalue()).decode(), img.size

# Recommended pattern: declare the tool with the scaled dimensions,
# so Claude returns coordinates in the same space you send screenshots in.
b64, (scaled_w, scaled_h) = screenshot_scaled_b64()

COMPUTER_TOOL = {
    "type": "computer_20250124", "name": "computer",
    "display_width_px": scaled_w, "display_height_px": scaled_h,
}

# When executing clicks, remember to translate scaled coordinates back to
# real display coordinates:
def to_real_coords(scaled_x: int, scaled_y: int, real_w: int, real_h: int,
                   scaled_w: int, scaled_h: int) -> tuple[int, int]:
    return int(scaled_x * real_w / scaled_w), int(scaled_y * real_h / scaled_h)