Video Q&A

import os

from perceptron import Client, video

client = Client(api_key=os.environ["PERCEPTRON_API_KEY"])
video_url = (
    "https://raw.githubusercontent.com/perceptron-ai-inc/perceptron/"
    "main/cookbook/_shared/assets/capabilities/video-qa/robot_assembly.mp4"
)
messages = [{
    "role": "user",
    "content": [
        video(video_url),
        {"type": "text", "text": (
            "Watch this robot-assembly episode. State the likely overall goal, "
            "then list the observed subgoals in chronological order. "
            "For each subgoal, describe the visible action that supports it. "
            "Distinguish an inferred intention from something directly visible. "
            "Finally, say whether the video shows the overall goal being "
            "completed, remaining incomplete, or an uncertain outcome."
        )},
    ],
}]
response = client.chat.completions.create(
    model="perceptron-mk1.5",
    messages=messages,
    reasoning_effort="high",
    max_completion_tokens=2048,
)
choice = response.choices[0]
if choice.finish_reason != "stop":
    raise RuntimeError(f"Incomplete video answer: {choice.finish_reason}")
answer = choice.message.content or ""
print(answer)

messages.extend([
    {"role": "assistant", "content": answer},
    {"role": "user", "content": (
        "Recheck the ending of the video. Which visible action most directly "
        "supports your outcome assessment? Describe it and cite its interval "
        "with a clip annotation for asset_idx=0, using explicit seconds. "
        "If the ending is inconclusive, explain what evidence is missing."
    )},
])
follow_up = client.chat.completions.create(
    model="perceptron-mk1.5",
    messages=messages,
    reasoning_effort="high",
    max_completion_tokens=2048,
    vision_config={"annotation_format": "clip"},
)
choice = follow_up.choices[0]
if choice.finish_reason != "stop":
    raise RuntimeError(f"Incomplete follow-up answer: {choice.finish_reason}")
print(choice.message.content or "")

response = client.chat.completions.create(
    model="perceptron-mk1.5",
    messages=[{
        "role": "user",
        "content": [
            video(
                "https://raw.githubusercontent.com/perceptron-ai-inc/perceptron/"
                "main/cookbook/_shared/assets/capabilities/video-clipping/mj_shot_short.mp4"
            ),
            {"type": "text", "text": (
                "What happens during the basketball play, and what does the soundtrack "
                "add to that account? Separate visible actions from audible speech or "
                "reactions. Mark unclear speech instead of guessing the words."
            )},
        ],
    }],
    reasoning_effort="high",
    max_completion_tokens=2048,
    vision_config={"enable_audio_in_video": True},
)
choice = response.choices[0]
if choice.finish_reason != "stop":
    raise RuntimeError(f"Incomplete audiovisual answer: {choice.finish_reason}")
print(choice.message.content or "")