Image Q&A

import os
from pathlib import Path

from perceptron import Client, image

client = Client(api_key=os.environ["PERCEPTRON_API_KEY"])
image_url = (
    "https://raw.githubusercontent.com/perceptron-ai-inc/perceptron/"
    "main/cookbook/_shared/assets/capabilities/qna/studio_scene.webp"
)
messages = [{
    "role": "user",
    "content": [
        image(image_url),
        {"type": "text", "text": (
            "What stands out in this scene? Describe the visible objects and their "
            "relationships. Cite the objects supporting your answer with flat point_box "
            "annotations (no collections or tracks), each with a mention and asset_idx=0. Distinguish what is "
            "visible from any interpretation, and say when a detail is unclear."
        )},
    ],
}]
response = client.chat.completions.create(
    model="perceptron-mk1.5",
    messages=messages,
    reasoning_effort="high",
    max_completion_tokens=2048,
    vision_config={"annotation_format": "box"},
)
choice = response.choices[0]
if choice.finish_reason != "stop" or choice.message.tool_calls:
    raise RuntimeError(f"Incomplete image answer: {choice.finish_reason}")
answer = choice.message.content or ""
print(answer)
Path("response.txt").write_text(answer, encoding="utf-8")

<point_box mention="object cited in the answer" asset_idx="0"> (100,150) (300,450) </point_box>

messages.append(choice.message.to_dict())
messages.append({
    "role": "user",
    "content": (
        "Which of those observations is most important to your interpretation? "
        "Explain briefly and cite its visible evidence with a flat point_box on "
        "asset_idx=0, without a collection or track wrapper."
    ),
})
follow_up = client.chat.completions.create(
    model="perceptron-mk1.5",
    messages=messages,
    reasoning_effort="high",
    max_completion_tokens=1024,
    vision_config={"annotation_format": "box"},
)
follow_up_choice = follow_up.choices[0]
if follow_up_choice.finish_reason != "stop" or follow_up_choice.message.tool_calls:
    raise RuntimeError("The follow-up answer did not complete")
print(follow_up_choice.message.content or "")