Image captioning

import os

from perceptron import Client, image

client = Client(api_key=os.environ["PERCEPTRON_API_KEY"])
image_url = (
    "https://raw.githubusercontent.com/perceptron-ai-inc/perceptron/"
    "main/cookbook/_shared/assets/capabilities/caption/suburban_street.webp"
)


def caption(prompt, grounded=False):
    response = client.chat.completions.create(
        model="perceptron-mk1.5",
        messages=[{
            "role": "user",
            "content": [
                image(image_url),
                {"type": "text", "text": prompt},
            ],
        }],
        reasoning_effort="high",
        max_completion_tokens=2048,
        vision_config={"annotation_format": "box"} if grounded else {},
    )
    choice = response.choices[0]
    if choice.finish_reason != "stop" or choice.message.tool_calls:
        raise RuntimeError(f"Incomplete caption: {choice.finish_reason}")
    return choice.message.content or ""


prompts = {
    "concise": (
        "Write one sentence of alt text identifying the main visible subjects and "
        "setting. Include only details supported by the image."
    ),
    "detailed": (
        "Describe this image in one detailed paragraph. Cover the main subjects, "
        "their positions, the foreground and background, and visible relationships. "
        "Do not invent events, identities, or details outside the image."
    ),
}
for style, prompt in prompts.items():
    print(f"{style}: {caption(prompt)}")

from pathlib import Path

grounded_caption = caption(
    "Describe the main visible subjects and their surroundings. Interleave the "
    "description with flat point_box annotations for the subjects you mention, "
    "without collections or tracks. Give each box a descriptive mention and "
    "asset_idx=0. Use normalized coordinates.",
    grounded=True,
)
print(grounded_caption)
Path("response.txt").write_text(grounded_caption, encoding="utf-8")

The scene includes <point_box mention="vehicle" asset_idx="0"> (100,400) (350,700) </point_box> beside the road.