Skip to content

AgentTask.kt

src/jvmMain/kotlin/afield/ai/agent/AgentTask.kt · 745 lines

package afield.ai.agent

import afield.document.graph.Box
import afield.app.protocol.payload
import afield.app.diagnostics.BoxOutputLog
import afield.runtime.execution.ExecutionReports
import afield.app.server.SessionInformation
import afield.ai.tools.Ai4BackdoorTools
import afield.ai.models.LlmBridge
import afield.ai.models.LlmSendResult
import afield.ai.models.ScriptedLlmBridge
import afield.ai.tools.ToolExecutionContext
import afield.ai.models.createLlmBridge
import afield.ai.models.createIdeBridge
import afield.document.graph.isTrue
import afield.runtime.execution.sourceRevision
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.put
import java.util.UUID

Headless agent tasks: a goal, a document and a set of checks, run through AI4's own turn and judged from what the document's boxes did.

AgentTaskRequest describes the task and its AgentChecks, AgentTaskRunner runs it, and AgentTaskResult carries the verdict, each check's AgentCheckOutcome and the tool calls the agent made (AgentToolCallRecord).

data class AgentTaskRequest

A whole task handed to Field's own agent: a goal, a document, a tool-round budget, and checks that decide — from the document and what its boxes said, not from the agent's final text — whether the goal was met.

This is the in-Field counterpart of an external agent driving tools/field_backdoor.py. The turn runs through the same LlmBridge loop as the AI4 chat panel, with the native graph tools (FieldGraphTools) wired, and needs no browser: AgentTaskRunner.run takes a SessionInformation and returns an AgentTaskResult.

body — 69 lines, from line 40 
@Serializable
data class AgentTaskRequest(
    val goal: String,
    /** Checks that decide the verdict. Empty = verdict `unchecked`. */
    val checks: List<AgentCheck> = emptyList(),
    /** Tool calls applied before the agent's turn (not counted against its budget or in its log). */
    val setup: String = "",
    /** Box the conversation is about (uid or name); optional. */
    val focusBox: String = "",
    val maxToolRounds: Int = 24,
    /** `live` = the configured provider; `scripted` = replay [script]. */
    val mode: String = "live",
    /** JSON script for `scripted` mode; see [ScriptedLlmBridge.parse]. */
    val script: String = "",
    /** Model override for `live` mode; blank = the configured flagship. */
    val model: String = "",
    /** Appended to the harness instructions. */
    val extraInstructions: String = "",
    val taskId: String = "",
    /**
     * Run the task *as* this box's AI4 panel turn (uid or unique name): it goes through the real
     * `ai-assist` handler with kind `general`, so the exchange is saved in the box's
     * `ai4_chat_history` and shows in the normal chat UI. Live mode only. The per-call tool log is
     * then not available here (the panel keeps it); the checks still decide the verdict.
     */
    val chatBox: String = "",
    /** `on` / `off` pins the eval-distilled lessons in the instructions ([AgentLessons]); blank = default. */
    val lessons: String = "",
    /** Withhold [ViewBoxOutputRequest] — the counterfactual for a visual task: can it be done without looking? */
    val blind: Boolean = false,
    /**
     * `off` withholds `RunToolProgramRequest` (the tool-program A/B arm); `on` or blank leaves it to
     * [afield.ai.tools.program.ToolPrograms.enabled].
     */
    val programs: String = "",
    /**
     * `legacy` sends this task's instructions the pre-2026-10-02 way, inside the stored message, instead of
     * as ephemeral per-request context ([afield.ai.conversation.TurnContext]): the context-pipeline A/B arm.
     */
    val context: String = "",
    /**
     * Further user turns, sent in order in the same conversation after the goal's turn finishes. The checks
     * run after the last. Any non-empty list makes the task multi-turn.
     */
    val followUps: List<String> = emptyList(),
    /** Tool simple names withheld from this task's turn (as [programs] `off` withholds one); see [AgentEvalRequest.sealed]. */
    val withhold: List<String> = emptyList(),
    /**
     * A checkout the `repo` tools ([afield.ai.tools.RepoFileTools]) work in: blank = no repo tools. [repoRead]
     * and [repoWrite] are globs over root-relative paths; the job may read only the first and write only the second.
     */
    val repoRoot: String = "",
    val repoRead: List<String> = emptyList(),
    val repoWrite: List<String> = emptyList(),
    /**
     * Names from [afield.ai.tools.RepoCommandTemplates] this job may run with `RepoRunCommandRequest`, in [repoRoot]
     * (empty = no command tool); [repoCommandMaxSec] caps any one run's wall clock.
     */
    val repoCommands: List<String> = emptyList(),
    val repoCommandMaxSec: Int = 900,
    /**
     * Whether a `live` turn may go to a provider other than `local` (OpenAI, OpenRouter, Anthropic), which
     * sends the goal, the document's code and every tool result — docs pages included — off this machine.
     * In-Field callers (the panel, box tasks) default to yes, since the user picked the provider. The backdoor
     * sends `false` unless `--allow-remote-provider` or `AFIELD_AGENT_TASK_ALLOW_REMOTE=1`, so an unattended
     * run cannot fall through to a remote model it was not told to use ([remoteProviderRefusal]).
     */
    val allowRemote: Boolean = true,
)

fun remoteProviderRefusal(mode: String, provider: String, allowRemote: Boolean): String?

The refusal for a live task that would reach a remote provider without opt-in, or null when it may run. Pure so the gate is testable without a provider.

fun remoteProviderRefusal(mode: String, provider: String, allowRemote: Boolean): String? =
    if (mode != "live" || allowRemote || provider == "local") null
    else "AI provider is '$provider', which sends the goal, code and tool results (docs pages included) off this machine. " +
        "Pass --allow-remote-provider (or set AFIELD_AGENT_TASK_ALLOW_REMOTE=1) to allow it, or select the 'local' provider."

data class AgentCheck

One predicate over the document. Kept as data (not a lambda) so a task can arrive over HTTP or the backdoor, and so a result can say exactly what was checked.

  • box-exists box
  • output-contains box text — printed or displayed since the agent's turn began (box blank = any box)
  • output-ints-include box ints — those integers appear, in order, in what it printed/displayed
  • property-contains / property-lacks box property text
  • last-run-ok box — its latest run/launch since the turn began succeeded, and was of its current code
  • connected parent child
  • running / not-running box
  • output-picture box — a capture of its output area succeeds and is not blank (≥ 1% of pixels are not background)
  • output-colour box text = #rrggbb fraction — at least that share of the capture is within 48 of that colour
  • output-rising box text = #c1,#c2,… fraction = minimum bar count — the bars drawn in those colours rise strictly left to right, and (with more than one colour) the last, tallest bar is mostly the last colour
  • effects-absent / effects-present text = a afield.ai.tools.ToolEffect — over every call the turn made, a program's nested calls included; a call with undeclared effects fails effects-absent
  • tool-used text = a tool name — the turn called it (directly or from a program), whatever the call returned
  • tool-ok text = a tool name — at least one such call succeeded (its AgentToolCallRecord.ok); the detail counts the failures. Use this, not tool-used, when the tool doing its job is the point: a run whose every call errored ("No tool registered") passes tool-used and fails this
@Serializable
data class AgentCheck(
    val kind: String,
    val box: String = "",
    val text: String = "",
    val property: String = "code",
    val parent: String = "",
    val child: String = "",
    val ints: List<Long> = emptyList(),
    val fraction: Double = 0.0,
) {

companion object

    companion object {

fun parse(spec: String): AgentCheck

The CLI spelling: kind:arg:arg. output-contains:Primes:2, 3, 5, connected:A:B, last-run-ok:Primes, output-ints-include:Primes:2,3,5,7, property-contains:Box:code:text, property-lacks:Box:code:text. Only the last argument may contain :.

        fun parse(spec: String): AgentCheck {
            val kind = spec.substringBefore(':').trim()
            val rest = spec.substringAfter(':', "")
            return when (kind) {
                "box-exists", "last-run-ok", "running", "not-running" -> AgentCheck(kind, box = rest)
                "output-contains" -> AgentCheck(kind, box = rest.substringBefore(':'), text = rest.substringAfter(':', ""))
                "output-ints-include" -> AgentCheck(kind, box = rest.substringBefore(':'),
                    ints = rest.substringAfter(':', "").split(',', ' ').mapNotNull { it.trim().toLongOrNull() })
                "property-contains", "property-lacks" -> {
                    val parts = rest.split(':', limit = 3)
                    require(parts.size == 3) { "$kind:<box>:<property>:<text>" }
                    AgentCheck(kind, box = parts[0], property = parts[1], text = parts[2])
                }
                "connected" -> AgentCheck(kind, parent = rest.substringBefore(':'), child = rest.substringAfter(':', ""))
                "output-picture" -> AgentCheck(kind, box = rest)
                "effects-absent", "effects-present" -> {
                    afield.ai.tools.ToolEffect.valueOf(rest.trim())
                    AgentCheck(kind, text = rest.trim())
                }
                "tool-used", "tool-ok" -> AgentCheck(kind, text = rest.trim())
                "output-colour" -> {
                    val parts = rest.split(':')
                    require(parts.size == 3) { "output-colour:<box>:#rrggbb:<fraction>" }
                    PictureStats.parseHex(parts[1])
                    AgentCheck(kind, box = parts[0], text = parts[1], fraction = parts[2].toDouble())
                }
                "output-rising" -> {
                    val parts = rest.split(':')
                    require(parts.size == 3) { "output-rising:<box>:#c1,#c2:<minBars>" }
                    parts[1].split(',').forEach { PictureStats.parseHex(it) }
                    AgentCheck(kind, box = parts[0], text = parts[1], fraction = parts[2].toDouble())
                }
                else -> throw IllegalArgumentException("Unknown check kind '$kind'")
            }
        }
    }
}

data class AgentCheckOutcome(val check: AgentCheck, val passed: Boolean, val detail: String)

@Serializable
data class AgentCheckOutcome(val check: AgentCheck, val passed: Boolean, val detail: String)

data class AgentToolCallRecord

@Serializable
data class AgentToolCallRecord(
    val index: Int,
    val tool: String,
    val arguments: String,
    /** The tool's own verdict: false when it threw or returned `success: false` / `error: true`. */
    val ok: Boolean,
    val result: String,
    /** The call this one ran inside — a tool program's — or blank for a call the model made itself. */
    val parent: String = "",
)

data class AgentTaskResult

body — 54 lines, from line 211 
@Serializable
data class AgentTaskResult(
    val taskId: String,
    val goal: String,
    val mode: String,
    val provider: String,
    val model: String,
    /** `pass`, `fail`, `budget-exhausted`, `error`, `unchecked`, `leaked` (a sealed corpus pass that read its solution), or `running` when observation ended before a live panel turn. */
    val verdict: String,
    val checks: List<AgentCheckOutcome>,
    val toolCalls: List<AgentToolCallRecord>,
    val finalText: String,
    val rounds: Int,
    val maxToolRounds: Int,
    val hitToolRoundCap: Boolean,
    val elapsedMs: Long,
    val inputTokens: Long? = null,
    val outputTokens: Long? = null,
    /** Input tokens the provider served from its prompt cache (see [afield.ai.models.LlmSendResult.cacheReadTokens]). */
    val cacheReadTokens: Long? = null,
    val cacheWriteTokens: Long? = null,
    val documentName: String,
    val createdBoxes: List<String> = emptyList(),
    val setupCalls: List<AgentToolCallRecord> = emptyList(),
    val error: String = "",
    /** Set when the task ran as a box's panel turn ([AgentTaskRequest.chatBox]). */
    val chatBox: String = "",
    /** Failed tool calls that were mistakes about the tools, not the box's own errors ([AgentLessons.isMisuse]). */
    val misuseCalls: Int = 0,
    /** Tool results that attached a picture the model was shown. */
    val imagesSeen: Int = 0,
    /** Whether the eval-distilled lessons were in this turn's instructions. */
    val lessons: Boolean = false,
    /** The document as file refs spell it (`afield://<documentRef>/<uid>/code`): its directory's name. */
    val documentRef: String = "",
    /** Whether `RunToolProgramRequest` was offered, how often the model called it, and the calls those programs made. */
    val programs: Boolean = false,
    val programCalls: Int = 0,
    val nestedCalls: Int = 0,
    /** User turns in the conversation: 1 + [AgentTaskRequest.followUps]. */
    val turns: Int = 1,
    /** Where the last turn's instructions went: `instructions`, `system+prefix`, or `inline` (the legacy arm). */
    val contextPlacement: String = "",
    /** A mined-corpus task only: the tool whose result held a line of the task's own solution (see [AgentEvalCorpus.leak]). */
    val leakedVia: String = "",
    /** Root-relative paths the `repo` tools wrote, in order (a path written twice appears twice). */
    val repoWritten: List<String> = emptyList(),
    /** The job's command runs, `name args -> exit code` (or `timeout`), in order. */
    val repoRuns: List<String> = emptyList(),
    /** Per tool name: calls and failed calls ([toolCallTally]), so an all-errored tool shows without reading [toolCalls]. */
    val toolTally: Map<String, ToolCallTally> = emptyMap(),
    /** True when the turn went to a provider other than `local`, i.e. its content left this machine. */
    val offMachine: Boolean = false,
)

data class ToolCallTally(val calls: Int, val failed: Int)

@Serializable
data class ToolCallTally(val calls: Int, val failed: Int)

fun toolCallTally(calls: List<AgentToolCallRecord>): Map<String, ToolCallTally>

fun toolCallTally(calls: List<AgentToolCallRecord>): Map<String, ToolCallTally> =
    calls.groupBy { it.tool }.mapValues { (_, c) -> ToolCallTally(c.size, c.count { !it.ok }) }

val HEADLESS_TASK_INSTRUCTIONS: String

What a headless task adds to the ordinary AI4 chat instructions (afield.ai.conversation.buildDynamicInstructions, which already describes the graph tools via afield.ai.conversation.fieldGraphToolGuidance). Only the facts that differ from a person typing in the chat: nobody will answer a question, and the turn is judged by checks.

val HEADLESS_TASK_INSTRUCTIONS: String = """
This turn was handed to you as a task, not typed by a person watching: nobody can answer questions.
Work through your tools until the goal is met, then answer with one or two sentences saying what you
did and what the tool results showed. The task is judged by checks on the document afterwards.
""".trimIndent()

object AgentTaskRunner

object AgentTaskRunner {

fun run

Run request against info. Blocking; call it off the Runloop (the tools dispatch onto it).

contextFor builds the tool context — by default the backdoor's full context (macros, guides, Kotlin tools) plus FieldGraphTools on host. bridgeFor builds the model bridge — by default ScriptedLlmBridge for scripted and createLlmBridge for live.

body — 134 lines, from line 293 
    fun run(
        request: AgentTaskRequest,
        info: SessionInformation,
        host: FieldGraphHost = LiveFieldGraphHost,
        contextFor: (SessionInformation, FieldGraphTools, String?) -> ToolExecutionContext = { i, graph, focus ->
            Ai4BackdoorTools.buildContext(i, focus).copy(fieldGraph = graph)
        },
        bridgeFor: (AgentTaskRequest) -> LlmBridge = ::defaultBridge,
    ): AgentTaskResult {
        val taskId = request.taskId.ifBlank { "task-" + UUID.randomUUID().toString().take(8) }
        val started = System.nanoTime()
        val graph = FieldGraphTools(info, host, lessons = AgentLessons.resolve(request.lessons), seesOutput = !request.blind)
        val budget = request.maxToolRounds.coerceIn(1, 64)
        var bridge: LlmBridge? = null
        var setupLog = emptyList<AgentToolCallRecord>()
        fun elapsed() = (System.nanoTime() - started) / 1_000_000

        fun failure(message: String, sent: LlmSendResult? = null) = AgentTaskResult(
            taskId = taskId, goal = request.goal, mode = request.mode,
            provider = bridge?.modelIdentity?.provider ?: "", model = bridge?.modelIdentity?.requestedModel ?: request.model,
            verdict = "error", checks = emptyList(), toolCalls = sent?.let(::records).orEmpty(),
            finalText = sent?.text.orEmpty(), rounds = sent?.rounds ?: 0, maxToolRounds = budget,
            hitToolRoundCap = sent?.hitToolRoundCap ?: false, elapsedMs = elapsed(),
            documentName = info.documentName, createdBoxes = graph.created.toList(), setupCalls = setupLog,
            error = message
        )

        return try {
            remoteProviderRefusal(request.mode, afield.ai.models.configuredAiProvider(), request.allowRemote)?.let { return failure(it) }
            val ctx = contextFor(info, graph, request.focusBox.ifBlank { null }).let { base ->
                val off = (if (request.programs == "off") setOf(afield.ai.tools.program.RUN_TOOL_PROGRAM) else emptySet()) + request.withhold
                if (off.isNotEmpty()) base.copy(withheld = base.withheld + off) else base
            }.let { base ->
                if (request.repoRoot.isBlank()) base else {
                    val root = java.io.File(request.repoRoot)
                    require(root.isDirectory) { "repoRoot '${request.repoRoot}' is not a directory" }
                    val files = afield.ai.tools.RepoFileTools(afield.ai.tools.RepoScope(root.toPath(),
                        request.repoRead.ifEmpty { listOf("**") }, request.repoWrite))
                    base.copy(repoFiles = files, repoCommands = request.repoCommands.takeIf { it.isNotEmpty() }?.let {
                        afield.ai.tools.RepoCommandTools(files, afield.ai.tools.RepoCommandScope(it.map(String::trim).toSet(),
                            maxTimeoutSec = request.repoCommandMaxSec.coerceIn(10, 3600)))
                    })
                }
            }

            if (request.setup.isNotBlank()) {
                val setup = ScriptedLlmBridge(ScriptedLlmBridge.parse(request.setup), "setup")
                val sent = setup.send(setup.startThread("setup", ""), "setup", ctx = ctx, maxToolRounds = 64)
                setupLog = records(sent)
                setupLog.firstOrNull { !it.ok && !it.tool.startsWith("RunBox") && !it.tool.startsWith("LaunchBox") }?.let {
                    // A setup run is allowed to fail — "fix the compile error" needs one — but a
                    // setup edit that fails means the task never existed.
                    return failure("Setup step ${it.index} (${it.tool}) failed: ${it.result.take(400)}")
                }
            }

            if (request.chatBox.isNotBlank()) return runInChatPanel(request, info, host, contextFor, bridgeFor, graph, setupLog, budget, taskId, started)

            val marker = BoxOutputLog.mark()
            val b = bridgeFor(request)
            bridge = b
            val extra = HEADLESS_TASK_INSTRUCTIONS +
                (if (request.extraInstructions.isNotBlank()) "\n\n" + request.extraInstructions else "")
            // The same thread instructions and the same chat turn as the AI4 document chat: this is a
            // headless entry into that turn, not a second harness.
            val thread = b.startThread("afield-task://${info.documentName}/$taskId", "",
                afield.ai.models.OpenAiIdeCodexBridge.defaultChatThreadInstructions)
            val prompt = buildString {
                appendLine("GOAL: ${request.goal}")
                appendLine()
                appendLine("Document: ${info.documentName}. Boxes now: " +
                    info.boxes.values.filter { !it.getPropertyRaw("_tombstone").isTrue() }
                        .joinToString(", ") { it.name.ifBlank { it.id } }.ifBlank { "(none)" })
                appendLine("You have at most $budget tool rounds.")
                ctx.repoFiles?.scope?.let { s ->
                    appendLine()
                    appendLine("Repository: ${s.root}. Work on its files with the repo tools (RepoReadFileRequest, RepoListFilesRequest, " +
                        "RepoSearchRequest, RepoPatchFileRequest, RepoWriteFileRequest); paths are relative to that root. " +
                        "Readable: ${s.readable.joinToString(", ")}. Writable: ${s.writable.joinToString(", ").ifBlank { "(nothing)" }}. " +
                        (if (ctx.repoCommands == null) "There is no shell: anything the goal says to run as a command, do through these tools instead."
                        else "There is no shell, but RepoRunCommandRequest runs these named commands in the root (argv templates; " +
                            "use them to reproduce a failure before you edit and to confirm the fix after):"))
                    ctx.repoCommands?.let { appendLine(it.describeAllowed()) }
                }
            }
            val helpers = afield.ai.conversation.IdeChatHelpers(b,
                ephemeralInstructions = request.context != "legacy" && afield.ai.conversation.ContextPipeline.enabledByDefault)
            var sent = helpers.chatGeneral(
                thread, code = "", lang = afield.runtime.languages.KOTLIN_SCRIPT, userPrompt = prompt, ctx = ctx,
                maxToolRounds = budget, extraInstructions = extra)
            for (followUp in request.followUps) {
                sent = mergeTurns(sent, helpers.chatGeneral(
                    sent.threadState, code = "", lang = afield.runtime.languages.KOTLIN_SCRIPT, userPrompt = followUp, ctx = ctx,
                    maxToolRounds = budget, extraInstructions = extra))
            }

            val shots = HashMap<String, BoxOutputShot>()
            val calls = records(sent)
            val outcomes = request.checks.map { evaluate(it, info, graph, marker, shots, calls, ctx) }
            val verdict = when {
                outcomes.isEmpty() -> "unchecked"
                outcomes.all { it.passed } -> "pass"
                sent.hitToolRoundCap -> "budget-exhausted"
                else -> "fail"
            }
            AgentTaskResult(
                taskId = taskId, goal = request.goal, mode = request.mode,
                provider = sent.modelIdentity?.provider ?: b.modelIdentity?.provider.orEmpty(),
                model = sent.modelIdentity?.let { it.responseModel?.takeIf { m -> m.isNotBlank() } ?: it.requestedModel } ?: b.modelIdentity?.requestedModel.orEmpty(),
                verdict = verdict, checks = outcomes, toolCalls = calls, finalText = sent.text,
                rounds = sent.rounds, maxToolRounds = budget, hitToolRoundCap = sent.hitToolRoundCap,
                elapsedMs = elapsed(), inputTokens = sent.inputTokens, outputTokens = sent.outputTokens,
                cacheReadTokens = sent.cacheReadTokens, cacheWriteTokens = sent.cacheWriteTokens,
                documentName = info.documentName, createdBoxes = graph.created.toList(), setupCalls = setupLog,
                misuseCalls = calls.count(AgentLessons::isMisuse),
                // Only the model's own calls: a picture a program's nested call took reached the model
                // only if the program show()ed it, and then it rode on the program's result.
                imagesSeen = sent.toolTranscript.count { it.parentCallId.isEmpty() && it.resultJson.contains("\"imageAttached\":true") },
                lessons = graph.lessons,
                documentRef = java.io.File(info.filename).name,
                programs = request.programs != "off" && afield.ai.tools.program.ToolPrograms.enabled,
                programCalls = calls.count { it.tool == afield.ai.tools.program.RUN_TOOL_PROGRAM },
                nestedCalls = calls.count { it.parent.isNotEmpty() },
                turns = 1 + request.followUps.size,
                contextPlacement = sent.contextReport?.placement ?: "inline",
                repoWritten = ctx.repoFiles?.written?.toList().orEmpty(),
                repoRuns = ctx.repoCommands?.runs?.toList().orEmpty(),
                toolTally = toolCallTally(calls),
                offMachine = request.mode == "live" && (sent.modelIdentity?.provider ?: b.modelIdentity?.provider.orEmpty()).let { it.isNotBlank() && it != "local" },
            )
        } catch (t: Throwable) {
            failure("${t.javaClass.simpleName}: ${t.message}")
        }
    }

The task as an AI4 panel turn in AgentTaskRequest.chatBox: its exchange is written into that box's ai4_chat_history, the panel shows it, and the person can carry on the conversation.

Live: the goal goes through the real ai-assist handler (kind general) — the same turn a person's message starts — and the verdict is appended. Scripted: the replay runs through the same tools as always, and the whole exchange, tool calls included, is written into the transcript, labelled as a scripted replay.

Fails fast, with the reason in the transcript and the result, when the box already has an AI turn running or when the handler does not start one (AI setup failed).

source — 68 lines, from line 440 
    private fun runInChatPanel(
        request: AgentTaskRequest, info: SessionInformation, host: FieldGraphHost,
        contextFor: (SessionInformation, FieldGraphTools, String?) -> ToolExecutionContext,
        bridgeFor: (AgentTaskRequest) -> LlmBridge, graph: FieldGraphTools,
        setupLog: List<AgentToolCallRecord>, budget: Int, taskId: String, started: Long,
    ): AgentTaskResult {
        fun elapsed() = (System.nanoTime() - started) / 1_000_000
        val box = graph.resolve(request.chatBox)
        fun refuse(message: String): AgentTaskResult {
            AgentTaskTranscript.appendError(info, box, request.goal, message)
            return AgentTaskResult(taskId = taskId, goal = request.goal, mode = request.mode, provider = "", model = "",
                verdict = "error", checks = emptyList(), toolCalls = emptyList(), finalText = "", rounds = 0,
                maxToolRounds = budget, hitToolRoundCap = false, elapsedMs = elapsed(), documentName = info.documentName,
                setupCalls = setupLog, error = message, chatBox = box.id)
        }
        if (box.getPropertyRaw(afield.ai.conversation.AiBoxConversation.RUNNING_PROPERTY) == true) {
            // The persisted flag can survive a JVM interruption. The ordinary chat-history
            // handler consults the server's live turns before recovering it; use that same path
            // instead of treating a saved flag as evidence of an active turn.
            val previous = afield.app.server.thisSessionInfo.get()
            afield.app.server.thisSessionInfo.set(info)
            try {
                afield.app.protocol.handle(afield.app.protocol.Message("ai-chat-history").apply {
                    payload = buildJsonObject { put("uid", box.id) }
                }) { }
            } finally {
                if (previous == null) afield.app.server.thisSessionInfo.remove() else afield.app.server.thisSessionInfo.set(previous)
            }
        }
        if (box.getPropertyRaw(afield.ai.conversation.AiBoxConversation.RUNNING_PROPERTY) == true)
            return refuse("'${box.name}' already has an AI turn running. Wait for it (or stop it) and ask again.")

        if (request.mode == "scripted") {
            val result = run(request.copy(chatBox = "", setup = ""), info, host, contextFor, bridgeFor)
                .let { it.copy(taskId = taskId, setupCalls = setupLog, chatBox = box.id, elapsedMs = elapsed()) }
            AgentTaskTranscript.appendScriptedTurn(info, box, request.goal, result)
            return result
        }
        require(request.mode == "live") { "mode must be 'live' or 'scripted', not '${request.mode}'" }

        val marker = BoxOutputLog.mark()
        val turn = runAsBoxChatTurn(info, box, request, budget)
        if (turn.running) return AgentTaskResult(
            taskId = taskId, goal = request.goal, mode = request.mode, provider = "ai4-panel", model = "",
            verdict = "running", checks = emptyList(), toolCalls = emptyList(), finalText = "", rounds = -1,
            maxToolRounds = budget, hitToolRoundCap = false, elapsedMs = elapsed(), documentName = info.documentName,
            setupCalls = setupLog, chatBox = box.id, documentRef = java.io.File(info.filename).name
        )
        if (turn.error != null) return refuse(turn.error)
        val shots = HashMap<String, BoxOutputShot>()
        val outcomes = if (turn.failure != null) emptyList() else request.checks.map { evaluate(it, info, graph, marker, shots) }
        val result = AgentTaskResult(
            taskId = taskId, goal = request.goal, mode = request.mode,
            provider = turn.identity?.provider ?: "ai4-panel", model = turn.identity?.let { it.responseModel ?: it.requestedModel }.orEmpty(),
            verdict = when {
                turn.failure != null -> "error"
                turn.hitToolRoundCap -> "budget-exhausted"
                outcomes.isEmpty() -> "unchecked"
                outcomes.all { it.passed } -> "pass"
                else -> "fail"
            },
            checks = outcomes, toolCalls = emptyList(), finalText = turn.reply, rounds = turn.rounds, maxToolRounds = budget,
            hitToolRoundCap = turn.hitToolRoundCap, elapsedMs = elapsed(), documentName = info.documentName,
            setupCalls = setupLog, chatBox = box.id, error = turn.failure.orEmpty()
        )
        AgentTaskTranscript.appendVerdict(info, box, result)
        return result
    }

internal data class PanelTurn(val reply: String, val identity: afield.ai.conversation.AiChatModelIdentity? = null, val error: String? = null,

    internal data class PanelTurn(val reply: String, val identity: afield.ai.conversation.AiChatModelIdentity? = null, val error: String? = null,
        val running: Boolean = false, val failure: String? = null, val rounds: Int = -1, val hitToolRoundCap: Boolean = false)

internal var panelTurnLimitMs: Long = 10 * 60_000L

How long a turn that did start may run before this gives up on it (it keeps running).

Dispatch the goal to the real ai-assist handler and wait for this turn to finish.

The handler marks the box running synchronously, before it launches the turn, so "not running right after dispatch" means it did not start (setup failed, or it refused) — reported at once, never waited on. The reply is taken from what the history gained after the transcript as it was before dispatch, and must follow a <<you>> block carrying this goal; anything else is an error, never another turn's reply.

source — 47 lines, from line 524 
    private fun runAsBoxChatTurn(info: SessionInformation, box: Box, request: AgentTaskRequest, budget: Int): PanelTurn {
        val runningKey = afield.ai.conversation.AiBoxConversation.RUNNING_PROPERTY
        val before = afield.ai.conversation.AiBoxConversation.read(box)
        val beforeSequence = afield.ai.conversation.AiBoxConversation.turnSequence(box)
        val assist = afield.app.protocol.AiAssist(
            uid = box.id,
            prompt = (box.getPropertyRaw("code") as? String).orEmpty(),
            // Just the goal: this is an ordinary panel turn the person can see and continue, so the
            // headless "nobody can answer" framing does not apply, and it would be shown to them as
            // their own message. Carry the task's bounded budget through the ordinary panel path.
            extraPrompt = request.goal,
            property = "code",
            kind = "general",
            maxToolRounds = budget,
        )
        val previous = afield.app.server.thisSessionInfo.get()
        afield.app.server.thisSessionInfo.set(info)
        try {
            afield.app.protocol.handle(afield.app.protocol.Message("ai-assist").payload(assist)) { }
        } finally {
            if (previous == null) afield.app.server.thisSessionInfo.remove() else afield.app.server.thisSessionInfo.set(previous)
        }
        if (afield.ai.conversation.AiBoxConversation.turnSequence(box) == beforeSequence) return PanelTurn("", error =
            "AI4 did not start a turn in '${box.name}': AI setup failed or no provider is configured (see Field's notifications and /welcome).")
        val deadline = System.currentTimeMillis() + panelTurnLimitMs
        while (box.getPropertyRaw(runningKey) == true) {
            // An observation limit neither stops the live turn nor establishes failure.
            // In particular, do not append an "Agent task not started" user turn into
            // the conversation while the original task is still using its tools.
            if (System.currentTimeMillis() > deadline) return PanelTurn("", running = true)
            Thread.sleep(200)
        }
        val history = afield.ai.conversation.AiBoxConversation.read(box)
        val added = if (history.startsWith(before)) history.substring(before.length) else {
            val key = request.goal.trim().take(60)
            val at = history.lastIndexOf("<<you>>").takeIf { it >= 0 && history.indexOf(key, it) >= 0 }
                ?: return PanelTurn("", error = "Could not find this task's turn in '${box.name}'s chat history.")
            history.substring(at)
        }
        if (!added.contains(request.goal.trim().take(60))) return PanelTurn("", error = "The chat history gained a turn that is not this task's.")
        val last = added.substringAfterLast("<<ai>>").trimStart('\n')
        val (identity, text) = afield.ai.conversation.AiChatModelIdentity.splitFromContent(last)
        return PanelTurn(text.substringBefore("<<progress").trim().take(4000), identity,
            failure = afield.ai.conversation.AiBoxConversation.turnFailure(box),
            rounds = afield.ai.conversation.AiBoxConversation.turnRounds(box),
            hitToolRoundCap = afield.ai.conversation.AiBoxConversation.turnCapped(box))
    }

fun defaultBridge(request: AgentTaskRequest): LlmBridge

    fun defaultBridge(request: AgentTaskRequest): LlmBridge = when (request.mode) {
        "scripted" -> {
            require(request.script.isNotBlank()) { "scripted mode needs a script" }
            ScriptedLlmBridge(ScriptedLlmBridge.parse(request.script))
        }
        "live" -> if (request.model.isNotBlank()) createLlmBridge(model = request.model) else createIdeBridge()
        else -> throw IllegalArgumentException("mode must be 'live' or 'scripted', not '${request.mode}'")
    }

Two consecutive turns of one conversation as one result: counts summed, the later turn's text and thread.

    private fun mergeTurns(a: LlmSendResult, b: LlmSendResult): LlmSendResult {
        fun sum(x: Long?, y: Long?) = if (x == null && y == null) null else (x ?: 0L) + (y ?: 0L)
        return b.copy(
            commentary = a.commentary + b.commentary,
            conversationAdditions = a.conversationAdditions + b.conversationAdditions,
            toolTranscript = a.toolTranscript + b.toolTranscript,
            rounds = a.rounds.coerceAtLeast(0) + b.rounds.coerceAtLeast(0),
            hitToolRoundCap = a.hitToolRoundCap || b.hitToolRoundCap,
            inputTokens = sum(a.inputTokens, b.inputTokens),
            outputTokens = sum(a.outputTokens, b.outputTokens),
            cacheReadTokens = sum(a.cacheReadTokens, b.cacheReadTokens),
            cacheWriteTokens = sum(a.cacheWriteTokens, b.cacheWriteTokens),
        )
    }

    private fun records(sent: LlmSendResult): List<AgentToolCallRecord> =
        sent.toolTranscript.mapIndexed { index, entry ->
            val node = runCatching { afield.ai.tools.aiToolMapper.readTree(entry.resultJson) }.getOrNull()
            val ok = node == null || !(node.get("success")?.asBoolean(true) == false || node.get("error")?.asBoolean(false) == true)
            AgentToolCallRecord(index + 1, entry.name, entry.argumentsJson.take(4000), ok, entry.resultJson.take(4000), parent = entry.parentCallId)
        }

checks

    private fun find(info: SessionInformation, token: String): Box? {
        val live = info.boxes.values.filter { !it.getPropertyRaw("_tombstone").isTrue() }
        return live.firstOrNull { it.id == token } ?: live.filter { it.name == token }.let { if (it.size == 1) it[0] else null }
    }

    private fun said(box: Box, marker: Long): List<BoxOutputLog.Entry> =
        BoxOutputLog.since(box, marker).entries.filter { it.kind == "out" || it.kind == "display" }

internal fun evaluate(check: AgentCheck, info: SessionInformation, graph: FieldGraphTools, marker: Long,

body — 131 lines, from line 614 
    internal fun evaluate(check: AgentCheck, info: SessionInformation, graph: FieldGraphTools, marker: Long,
                          shots: MutableMap<String, BoxOutputShot> = HashMap(),
                          calls: List<AgentToolCallRecord>? = null,
                          ctx: ToolExecutionContext? = null): AgentCheckOutcome {
        fun pass(detail: String) = AgentCheckOutcome(check, true, detail)
        fun fail(detail: String) = AgentCheckOutcome(check, false, detail)
        fun box(token: String): Box? = find(info, token)
        // One capture per box per verdict: several picture checks on one box look at the same pixels.
        fun shot(b: Box): BoxOutputShot = shots.getOrPut(b.id) { graph.captureOutput(b, settleMs = 500) }
        // The effects a call declared (ToolMeta.kt), looked up in the shared registry plus this task's box tools.
        fun effectsOf(call: AgentToolCallRecord): Set<afield.ai.tools.ToolEffect> =
            (afield.ai.tools.Ai4BackdoorTools.registry.metaOf(ctx ?: ToolExecutionContext(), call.tool) ?: afield.ai.tools.ToolMeta.DEFAULT).effects
        return when (check.kind) {
            "effects-absent", "effects-present" -> {
                val log = calls ?: return fail("no per-call tool log for this task (a --chat-box task keeps it in the panel)")
                val effect = afield.ai.tools.ToolEffect.valueOf(check.text)
                val having = log.filter { effect in effectsOf(it) }
                if (check.kind == "effects-present") {
                    if (having.isNotEmpty()) pass("${having.size} call(s) with $effect, first ${having.first().tool}")
                    else fail("no call had $effect (${log.size} calls)")
                } else {
                    val undeclared = log.filter { afield.ai.tools.ToolEffect.UNDECLARED in effectsOf(it) }
                    when {
                        having.isNotEmpty() -> fail("${having.size} call(s) with $effect: " + having.take(3).joinToString { it.tool })
                        undeclared.isNotEmpty() -> fail("undeclared effects, so $effect cannot be ruled out: " + undeclared.take(3).joinToString { it.tool })
                        else -> pass("no call had $effect (${log.size} calls)")
                    }
                }
            }
            "tool-used" -> {
                val log = calls ?: return fail("no per-call tool log for this task")
                val hit = log.firstOrNull { it.tool == check.text || it.tool == check.text + "Request" }
                if (hit != null) pass("call ${hit.index} was ${hit.tool}") else fail("${check.text} was never called (${log.size} calls)")
            }
            "tool-ok" -> {
                val log = calls ?: return fail("no per-call tool log for this task (a --chat-box task keeps it in the panel)")
                val mine = log.filter { it.tool == check.text || it.tool == check.text + "Request" }
                val good = mine.firstOrNull { it.ok }
                val failed = mine.count { !it.ok }
                when {
                    mine.isEmpty() -> fail("${check.text} was never called (${log.size} calls)")
                    good == null -> fail("all ${mine.size} call(s) to ${check.text} failed, first: ${mine.first().result.take(200)}")
                    else -> pass("call ${good.index} succeeded; ${mine.size - failed} of ${mine.size} ok")
                }
            }
            "box-exists" -> box(check.box)?.let { pass("found ${it.id}") } ?: fail("no unique live box '${check.box}'")
            "output-contains" -> {
                val boxes = if (check.box.isBlank()) info.boxes.values.toList() else listOfNotNull(box(check.box))
                if (boxes.isEmpty()) return fail("no unique live box '${check.box}'")
                val hit = boxes.firstOrNull { b -> said(b, marker).any { it.text.contains(check.text) } }
                if (hit != null) pass("'${hit.name}' said it")
                else fail("not in output since the turn began: " +
                    boxes.flatMap { said(it, marker) }.takeLast(3).joinToString(" | ") { it.text.take(160) })
            }
            "output-ints-include" -> {
                val b = box(check.box) ?: return fail("no unique live box '${check.box}'")
                val texts = said(b, marker).map { it.text }
                val found = texts.any { text ->
                    val ints = Regex("-?\\d+").findAll(text).map { it.value.toLong() }.toList()
                    var i = 0
                    for (n in ints) if (i < check.ints.size && n == check.ints[i]) i++
                    i == check.ints.size
                }
                if (found) pass("found ${check.ints} in order")
                else fail("${check.ints} not found in order in: " + texts.takeLast(3).joinToString(" | ") { it.take(160) })
            }
            "property-contains" -> {
                val b = box(check.box) ?: return fail("no unique live box '${check.box}'")
                val value = b.getPropertyRaw(check.property) as? String ?: return fail("no text property ${check.property}")
                if (value.contains(check.text)) pass("contains it") else fail("${check.property} does not contain '${check.text}'")
            }
            "property-lacks" -> {
                val b = box(check.box) ?: return fail("no unique live box '${check.box}'")
                val value = b.getPropertyRaw(check.property) as? String ?: ""
                if (!value.contains(check.text)) pass("does not contain it") else fail("${check.property} contains '${check.text}'")
            }
            "last-run-ok" -> {
                val b = box(check.box) ?: return fail("no unique live box '${check.box}'")
                val verdicts = BoxOutputLog.since(b, marker).entries.filter { it.kind == "verdict" }
                val last = verdicts.lastOrNull() ?: return fail("not run since the turn began")
                if (last.success != true) return fail("latest run failed: ${last.text.take(300)}")
                val report = ExecutionReports.latest(b, "code")
                val current = (b.getPropertyRaw("code") as? String).orEmpty()
                if (report != null && report.revision != sourceRevision(current)) fail("its code changed after the successful run")
                else pass("latest run succeeded (${last.text.lineSequence().first()})")
            }
            "connected" -> {
                val p = box(check.parent) ?: return fail("no unique live box '${check.parent}'")
                val c = box(check.child) ?: return fail("no unique live box '${check.child}'")
                if (c in p.children) pass("'${p.name}' -> '${c.name}'") else fail("'${c.name}' is not a child of '${p.name}'")
            }
            "running" -> box(check.box)?.let { if (graph.isRunning(it)) pass("running") else fail("not running") }
                ?: fail("no unique live box '${check.box}'")
            "not-running" -> box(check.box)?.let { if (!graph.isRunning(it)) pass("not running") else fail("still running") }
                ?: fail("no unique live box '${check.box}'")
            "output-picture" -> {
                val b = box(check.box) ?: return fail("no unique live box '${check.box}'")
                val s = shot(b)
                val image = s.image ?: return fail("no picture: ${s.message}")
                val stats = PictureStats.of(image)
                val detail = "${image.width}x${image.height}, ink ${"%.3f".format(stats.inkFraction)}, ${stats.describeDominant()}" +
                    (if (s.path.isNotBlank()) ", ${s.path}" else "")
                if (stats.inkFraction >= 0.01) pass(detail) else fail("blank: $detail")
            }
            "output-colour" -> {
                val b = box(check.box) ?: return fail("no unique live box '${check.box}'")
                val s = shot(b)
                val image = s.image ?: return fail("no picture: ${s.message}")
                val share = PictureStats.coverage(image, check.text)
                val detail = "${"%.3f".format(share)} of ${image.width}x${image.height} is ${check.text} (need ${check.fraction}); " +
                    PictureStats.of(image).describeDominant()
                if (share >= check.fraction) pass(detail) else fail(detail)
            }
            "output-rising" -> {
                val b = box(check.box) ?: return fail("no unique live box '${check.box}'")
                val s = shot(b)
                val image = s.image ?: return fail("no picture: ${s.message}")
                val colours = check.text.split(',').map { it.trim() }
                val bars = PictureStats.bars(image, colours)
                val detail = "${bars.size} bar(s), heights ${bars.map { it.height }}, colours ${bars.map { colours[it.colour] }}"
                when {
                    bars.size < check.fraction.toInt() -> fail("need ${check.fraction.toInt()} bars: $detail")
                    bars.zipWithNext().any { (l, r) -> r.height <= l.height } -> fail("not rising left to right: $detail")
                    colours.size > 1 && bars.last().colour != colours.lastIndex -> fail("the tallest bar is not ${colours.last()}: $detail")
                    else -> pass(detail)
                }
            }
            else -> fail("unknown check kind '${check.kind}'")
        }
    }
}