FieldEncodeProcess.kt

src/jvmMain/kotlin/afield/media/stream/FieldEncodeProcess.kt · 246 lines

package afield.media.stream

import afield.app.diagnostics.logging.lprintln
import org.json.JSONArray
import org.json.JSONObject
import java.io.BufferedWriter
import java.io.File
import java.util.concurrent.TimeUnit
import kotlin.concurrent.thread

The JVM side of the field-encode helper, which encodes Field render targets (VideoToolbox on macOS, the field-encode-linux build on Linux) and streams them to the visionOS app.

FieldEncodeProcess starts the helper and exchanges newline-delimited JSON with it; image data never passes through this pipe. FieldRTTStreamer is the usual way to drive it from a box.

class FieldEncodeProcess

Launches and drives the field-encode Swift co-process (native/field-encode) — the macOS VideoToolbox sender for the visionOS stereo/flat docks (see dev_docs/vision_frontend.md -> "The sender seam" and native/field-encode/WIRE_FORMAT.md).

This is pure JVM glue: it spawns the binary, writes newline-delimited JSON commands to its stdin, and reads JSON events from its stdout. Pixels never cross this pipe — the helper attaches to Field's RTT by global IOSurfaceID (passed once in open) and streams HEVC straight to the device over UDP. Field's only jobs are: hand over the surface IDs, and tick frame once each eye's render fence has signalled.

The child is stateless and disposable: if it dies, call start again and re-open.

class FieldEncodeProcess(
    private val binaryPath: String = defaultBinaryPath(),
    private val workingDirectory: String? = null,
) {
    @Volatile
    private var process: Process? = null

    @Volatile
    private var stdin: BufferedWriter? = null

var running: Boolean

    @Volatile
    var running: Boolean = false
        private set

var lastEvent: JSONObject?

    @Volatile
    var lastEvent: JSONObject? = null
        private set

var onEvent: ((JSONObject) -> Unit)? = null

Invoked for every JSON event line on the helper's stdout (opened/encoded/error/...).

var onExit: ((Int) -> Unit)? = null

Invoked once when the process exits, with its exit code.

fun start(): Boolean

    @Synchronized
    fun start(): Boolean {
        if (running) return true
        val bin = File(binaryPath)
        if (!bin.canExecute()) {
            lprintln("error,field-encode",
                "binary not executable: $binaryPath — build it: (cd native/field-encode && swift build -c release)")
            return false
        }
        val proc = try {
            ProcessBuilder(binaryPath).apply {
                workingDirectory?.let { directory(File(it)) }
            }.start()
        } catch (t: Throwable) {
            lprintln("error,field-encode", "failed to start: ${t.message ?: t}")
            return false
        }
        process = proc
        stdin = proc.outputStream.bufferedWriter()
        running = true

        thread(name = "field-encode-stdout", isDaemon = true) {
            runCatching {
                proc.inputStream.bufferedReader().useLines { lines ->
                    lines.forEach { line ->
                        val trimmed = line.trim()
                        if (trimmed.isEmpty()) return@forEach
                        runCatching { JSONObject(trimmed) }.getOrNull()?.let { ev ->
                            lastEvent = ev
                            onEvent?.invoke(ev)
                        }
                    }
                }
            }
        }
        thread(name = "field-encode-stderr", isDaemon = true) {
            runCatching {
                proc.errorStream.bufferedReader().useLines { lines ->
                    lines.forEach { if (it.isNotBlank()) lprintln("info,field-encode", it) }
                }
            }
        }
        thread(name = "field-encode-wait", isDaemon = true) {
            val code = runCatching { proc.waitFor() }.getOrDefault(-1)
            running = false
            onExit?.invoke(code)
        }
        return true
    }

fun alloc(width: Int, height: Int, count: Int)

Allocate a ring of count IOSurfaces in the helper and get their global IDs back (via an {"ev":"allocated","ids":[…]} event on onEvent). Field imports these by ID as its RTT backing — they carry a real pixelFormat, unlike MoltenVK's export. Precedes open.

    fun alloc(width: Int, height: Int, count: Int) = send(
        JSONObject().put("cmd", "alloc")
            .put("width", width).put("height", height).put("count", count)
    )

fun open

Set up the encoder + UDP stream to dest ("host:port"). On macOS, pass surfaces (global IOSurfaceIDs) to look them up in the helper (legacy); omit to use the ring created by a prior alloc. On Linux, pass fdSurfaces + fdSocket — the helper has no alloc (it can't allocate a Vulkan image itself), and a raw fd number can't travel over JSON (it's per-process, not a system-wide handle like an IOSurfaceID): the real fds cross over an SCM_RIGHTS handoff on fdSocket instead (see afield.graphics.texture.UnixFdPass — caller sends them right after this command), and only size/rowPitch travel on the wire, in the same order as the fds sent over the socket.

    fun open(
        width: Int, height: Int, fps: Int, bitrateKbps: Int,
        dest: String, codec: String = "hevc", idrIntervalMs: Int = 1000,
        surfaces: List<Long> = emptyList(),
        fdSurfaces: List<afield.graphics.texture.VulkanExternalMemory.FdHandle> = emptyList(),
        fdSocket: String? = null,
        stereo: Boolean = false,
    ) = send(
        JSONObject()
            .put("cmd", "open")
            .apply {
                if (fdSurfaces.isNotEmpty()) {
                    put("surfaces", JSONArray(fdSurfaces.map {
                        JSONObject().put("size", it.size).put("rowPitch", it.rowPitch)
                    }))
                    fdSocket?.let { put("fdSocket", it) }
                } else if (surfaces.isNotEmpty()) {
                    put("surfaces", JSONArray(surfaces))
                }
            }
            .put("width", width).put("height", height).put("fps", fps)
            .put("bitrateKbps", bitrateKbps).put("dest", dest)
            .put("codec", codec).put("idrIntervalMs", idrIntervalMs)
            .put("stereo", stereo)
    )

fun frame

Signal that ring surface surface is rendered and ready to encode now.

poseFrameId and poseDisplayTimeNanos identify the head pose this frame was rendered from, for the wire's version-2 pose tail. Both or neither: half a pose sets the wire's POSE_VALID flag over a zero timestamp, and the receiver would reproject to the wrong instant — worse than carrying no pose at all. Omitting them is normal (the desktop and OpenXR paths have no uplinked pose) and the helper then clears the flag.

A helper binary older than the pose tail decodes into properties it does not have and silently emits version-1 headers — which the receiver still accepts, so the symptom is a stream with no pose rather than a broken one. Rebuild the helper if a pose you set never arrives.

    fun frame(
        surface: Int, seq: Long, ptsMicros: Long, forceIdr: Boolean = false,
        poseFrameId: Long? = null, poseDisplayTimeNanos: Long? = null,
    ) = send(
        JSONObject().put("cmd", "frame").put("surface", surface)
            .put("seq", seq).put("ptsMicros", ptsMicros).put("forceIdr", forceIdr)
            .also { json ->
                if (poseFrameId != null && poseDisplayTimeNanos != null) {
                    json.put("poseFrameId", poseFrameId)
                        .put("poseDisplayTimeNanos", poseDisplayTimeNanos)
                }
            }
    )

fun bitrate(kbps: Int) = send(JSONObject().put("cmd", "bitrate").put("kbps", kbps))

fun selftest

Fire the helper's self-contained smoke test (synthetic frames, no RTT needed).

    fun selftest(
        dest: String, frames: Int = 120, width: Int = 1280, height: Int = 720,
        fps: Int = 60, bitrateKbps: Int = 20000, codec: String = "hevc",
    ) = send(
        JSONObject().put("cmd", "selftest").put("dest", dest).put("frames", frames)
            .put("width", width).put("height", height).put("fps", fps)
            .put("bitrateKbps", bitrateKbps).put("codec", codec)
    )

fun send(cmd: JSONObject)

    fun send(cmd: JSONObject) {
        val writer = stdin ?: run {
            lprintln("warn,field-encode", "not running; dropped ${cmd.optString("cmd")}")
            return
        }
        synchronized(writer) {
            runCatching { writer.write(cmd.toString()); writer.write("\n"); writer.flush() }
                .onFailure { lprintln("warn,field-encode", "write failed: ${it.message}") }
        }
    }

fun close()

Graceful: send close, then close stdin (helper exits its readLine loop), then reap.

    fun close() {
        runCatching { send(JSONObject().put("cmd", "close")) }
        stop()
    }

fun stop()

    @Synchronized
    fun stop() {
        val proc = process ?: return
        runCatching { stdin?.close() }
        runCatching {
            proc.destroy()
            if (!proc.waitFor(300, TimeUnit.MILLISECONDS)) {
                proc.destroyForcibly()
                proc.waitFor(300, TimeUnit.MILLISECONDS)
            }
        }
        process = null
        stdin = null
        running = false
    }

companion object

    companion object {

fun defaultBinaryPath(): String

Picks the per-OS sender binary (same control+wire protocol either way — see WIRE_FORMAT.md). Linux builds one CMake target (native/field-encode-linux/build/); macOS prefers the SwiftPM release build, falling back to debug. Paths are relative to the field4 repo root (cwd).

        fun defaultBinaryPath(): String {
            if (afield.graphics.vulkan.util.OS == afield.graphics.vulkan.util.OSType.LINUX) {
                return File("native/field-encode-linux/build/field-encode-linux").path
            }
            val base = "native/field-encode/.build"
            val release = File("$base/release/field-encode")
            val debug = File("$base/debug/field-encode")
            return when {
                release.canExecute() -> release.path
                debug.canExecute() -> debug.path
                else -> release.path
            }
        }
    }
}