FieldRTTStreamer.kt

    private fun startMac(): Boolean {
        // 1. helper allocates a ring of properly-formatted IOSurfaces and returns their global ids.
        encoder.alloc(width = width, height = height, count = maxOf(1, ringSize))
        if (!allocLatch.await(5, TimeUnit.SECONDS)) {
            lprintln("error,field-stream", "timed out waiting for surface allocation")
            return false
        }
        val ids = allocatedIds.get()
        if (ids.size != maxOf(1, ringSize)) {
            lprintln("error,field-stream", "expected ${maxOf(1, ringSize)} surfaces, got ${ids.size}")
            return false
        }

        /*
        2. import each surface (looked up by id) as the backing of its own RTT — or, on KosmicKrisp,
        which cannot back an image with an IOSurface, give each an ordinary RTT and the surface's
        pages as a buffer that [render] copies the finished frame into (see IOSurfaceHostBuffer).
        */
        val viaHostBuffer = !renderer.device.supportsMetalObjects && renderer.device.supportsExternalMemoryHost
        for (id in ids) {
            val ref = MetalIOSurface.lookup(id)
            if (ref == 0L) {
                lprintln("error,field-stream", "IOSurfaceLookup failed for id $id")
                return false
            }
            val rtt = RenderToTexture(renderer) {
                if (!viaHostBuffer) importMetalIOSurface = ref
                this.width = this@FieldRTTStreamer.width
                this.height = this@FieldRTTStreamer.height
                format = colorFormat
                useDepth = false
                colorSampled = true // so the box can also sample it to screen if it wants
                colorTransferSrc = true
            }
            surfaces += rtt
            hostBuffers += if (viaHostBuffer) IOSurfaceHostBuffer(renderer.device, ref, readOnly = false) else null
        }
        if (viaHostBuffer)
            lprintln(
                "info,field-stream",
                "IOSurface ring via host-pointer buffers (${renderer.device.physicalDevice.driverName})",
            )

        // 3. open the stream (surfaces already live in the helper from alloc).
        encoder.open(
            width = width,
            height = height,
            fps = fps,
            bitrateKbps = bitrateKbps,
            dest = dest,
            codec = codec,
            idrIntervalMs = idrIntervalMs,
            stereo = stereo,
        )
        return true
    }

    private fun startLinux(): Boolean {
        for (i in 0 until maxOf(1, ringSize)) {
            val rtt = RenderToTexture(renderer) {
                exportOpaqueFd = true
                this.width = this@FieldRTTStreamer.width
                this.height = this@FieldRTTStreamer.height
                format = colorFormat
                useDepth = false
                colorSampled = true
            }
            surfaces += rtt
        }
        val handles = surfaces.map { rtt ->
            val image =
                rtt.colorImageRefs.getOrNull(0)
                    ?: run {
                        lprintln("error,field-stream", "ring surface has no color image to export")
                        return false
                    }
            VulkanExternalMemory.handle(image)
        }

        /*
        A raw fd number means nothing to another process by value alone (unlike a Mac
        IOSurfaceID) — hand the real kernel objects to the helper via SCM_RIGHTS on a fresh
        Unix socket, right after telling it (over the JSON channel) to expect them there.
        */
        val fdSocket = java.nio.file.Files.createTempFile("field-encode-", ".sock").toString()
        java.io.File(fdSocket).delete() // bind() requires the path not exist yet
        encoder.open(
            width = width,
            height = height,
            fps = fps,
            bitrateKbps = bitrateKbps,
            dest = dest,
            codec = codec,
            idrIntervalMs = idrIntervalMs,
            fdSurfaces = handles,
            fdSocket = fdSocket,
            stereo = stereo,
        )

        // The helper only starts listening once it processes the `open` line above; retry the
        // connect+send briefly to absorb that race rather than assume perfect ordering.
        val fds = handles.map { it.fd }.toIntArray()
        var sent = false
        val deadline = System.nanoTime() + 2_000_000_000L
        while (!sent && System.nanoTime() < deadline) {
            sent = UnixFdPass.sendFds(fdSocket, fds)
            if (!sent) Thread.sleep(20)
        }
        if (!sent) {
            lprintln("error,field-stream", "failed to hand fds to field-encode-linux over $fdSocket")
            return false
        }
        return true
    }

    override fun render(command: CommandBuffer, index: Int) {
        if (!streaming || surfaces.isEmpty()) return

        /*
        Emit at the configured `fps`, NOT at the renderer's rate. Field renders ~60-90fps; ticking
        every frame floods the transport, and a receiver that can't keep up (e.g. the sim's
        software decoder) drops fragments → P-frames decode against missing references → garbage
        until the next IDR.
        */
        val now = System.nanoTime()
        val intervalNanos = 1_000_000_000L / maxOf(1, fps)
        if (lastSentNanos == 0L) {
            lastSentNanos = now
        } else {
            val due = lastSentNanos + intervalNanos
            if (now < due) return
            /*
            Advance on the IDEAL grid, not to `now`. We can only emit on a renderer tick, so
            `now` is always a little past `due`; assigning `now` carries that rounding error
            forward and re-rounds it every frame. The result beats against the render rate —
            at 30fps target on a ~64fps renderer the gate alternates between 2 and 3 ticks
            (31ms / 47ms), averaging ~27fps with visible judder even though nothing is lost.
            Advancing by exactly one interval keeps the long-run rate on target and bounds the
            jitter to a single render tick. If we've fallen more than a whole interval behind
            (a real stall — GPU hitch, encoder backpressure), resync to `now` rather than
            bursting several frames to catch up.
            Advance along the grid in WHOLE intervals, never to `now`. Snapping to `now` on a late
            tick silently discards the remainder, which is why the long-run rate measured 29.56fps
            against a 30fps target. That 0.44fps deficit is fatal downstream: the headset displays
            at 90Hz, so a frame occupies a whole number of refreshes and 30.00fps is exactly 3.
            At 29.56fps each frame wants 33.83ms — 0.5ms more than 3 refreshes — so the error
            accumulates until one frame is held for 4 refreshes instead of 3, roughly every 22
            frames (~0.7s). No amount of receiver buffering can fix that; the rate has to be
            exact. Skipping whole intervals keeps both the phase and the long-run rate correct.
            */
            lastSentNanos = if (now - due > intervalNanos) {
                due + ((now - due) / intervalNanos) * intervalNanos
            } else due
        }

        /*
        Render the box's content into exactly the active ring surface, then encode it. Rendering
        only this one (not all N) is what makes the ring real: while the encoder reads slot N, the
        next tick renders slot N+1, so a surface isn't overwritten for `ringSize` tick-intervals.
        This target lives in `others` (rendered outside the main pass), so driving the RTT's own
        render pass here is the same call path as registering it in the render list.
        */
        val slot = (seq % surfaces.size).toInt()
        surfaces[slot].render(command, index)
        // KosmicKrisp: the RTT is not the surface, so copy the frame into it on this same command
        // buffer; the fence-gated `frame` tick below then covers the copy too.
        hostBuffers.getOrNull(slot)?.let { host ->
            surfaces[slot].recordColorTransferSource(command) { image ->
                host.copyFromImage(command.vkCommandBuffer, image, 4)
            }
        }

        val s = seq
        /*
        Presentation timestamp on the IDEAL grid, derived from the frame index — NOT from `now`.

        `now` is when a render tick happened to fire, and we can only emit on a tick (~8.3ms at
        Field's ~120fps loop), so a wall-clock pts bakes the quantisation error into the stream:
        a 30fps target lands 4 ticks apart most of the time but 3 (25ms) or 5 (41.7ms) whenever
        phase drifts — measured on loopback as ~190 of 2530 frames off-nominal, one every 0.45s.
        A receiver scheduling against those timestamps faithfully REPRODUCES the irregularity, so
        its jitter buffer can absorb delivery jitter but not this. Stamping `seq * interval` makes
        the timeline describe the cadence we intended; the receiver's buffer then has something
        regular to schedule against, and delivery jitter is what it soaks up.
        */
        val pts = s * 1_000_000L / maxOf(1, fps)
        val forceIdr = s == 0L
        /*
        Read the pose here, not in the deferred block: this is the tick that rendered the surface,
        so this is the pose it was drawn from. Reading it after the graphics frame completes would
        pick up whatever had arrived by then — a different pose, silently.
        */
        val pose = poseTag?.invoke()
        // Time the host's own stages. Recorded here rather than inside the deferred block for the
        // same reason the pose is read here: this is the tick that rendered the surface.
        afield.graphics.visionos.visionPoseLatest()?.let { afield.graphics.visionos.VisionPoseLatency.latched(s, it) }
        renderer.afterCurrentGraphicsFrameComplete {
            afield.graphics.visionos.VisionPoseLatency.submitted(s)
            encoder.frame(
                surface = slot,
                seq = s,
                ptsMicros = pts,
                forceIdr = forceIdr,
                poseFrameId = pose?.frameId,
                poseDisplayTimeNanos = pose?.displayTimeNanos,
            )
        }
        seq++
    }

    private fun onEncoderEvent(ev: JSONObject) {
        when (ev.optString("ev")) {
            "allocated" -> {
                val arr = ev.optJSONArray("ids")
                val ids = (0 until (arr?.length() ?: 0)).map { arr!!.getLong(it) and 0xFFFF_FFFFL }
                allocatedIds.set(ids)
                allocLatch.countDown()
            }
            "error" -> {
                lprintln("error,field-stream", "encoder: ${ev.optString("msg")}")
                allocLatch.countDown() // unblock start() so it fails fast
            }
            "opened" -> lprintln("info,field-stream", "encoder opened")
            // Closes the host's last span: submit (which waits the render fence) to encode complete.
            "encoded" -> {
                framesEncoded++
                afield.graphics.visionos.VisionPoseLatency.encoded(ev.optLong("seq", -1L))
            }
            else -> {}
        }
    }
}