Skip to content

VisionPoseUplinkSmokeTest.kt

src/jvmMain/kotlin/afield/examples/VisionPoseUplinkSmokeTest.kt · 257 lines

package afield.examples

import afield.app.protocol.Handled
import afield.graphics.visionos.VISION_POSE_CHANNEL
import afield.graphics.visionos.VISION_POSE_VERSION
import afield.graphics.visionos.VisionPoseStream
import afield.graphics.visionos.VisionPoseWireFrame
import afield.graphics.visionos.VisionPoseWireView
import afield.graphics.visionos.installVisionPoseHandler
import afield.graphics.visionos.visionPoseRawHandler
import afield.graphics.visionos.visionPoseWireJson
import afield.math.linalg.Matrix4f
import afield.math.linalg.Vector3f
import kotlin.math.abs

FIELD_TEST id: visionpose.uplink runner: jvm-main entrypoint: afield.examples.VisionPoseUplinkSmokeTest tags: [smoke, visionos, pose] timeoutSec: 90 expect: exitCode: 0 stdoutContains: - "VisionPoseUplinkSmokeTest ok"

object VisionPoseUplinkSmokeTest

Guards the pose uplink's arithmetic, which is the half of the head-tracking milestone that a device cannot check for you.

With a headset on, a wrong handedness, a transposed matrix or a swapped eye all look the same — "the world is wrong somehow" — and none of them look wrong in a screenshot. So the composition from the wire ((originFromDevice * deviceFromEye).inverse) and the tangent extraction out of the compositor's projection are pinned here against hand-computed values, and the hand-written JSON below is what the visionOS mirror is checked against: it is the format as documented, not as the Kotlin serializer happens to spell it today.

object VisionPoseUplinkSmokeTest {

Head at (1, 1.5, -2), yawed 90° about +Y. Column-major, as the wire carries it.

    private val originFromDevice = floatArrayOf(
        0f, 0f, -1f, 0f,
        0f, 1f, 0f, 0f,
        1f, 0f, 0f, 0f,
        1f, 1.5f, -2f, 1f,
    )

device→eye: a 64 mm interpupillary offset, no rotation.

    private fun eyeTransform(offsetX: Float) = floatArrayOf(
        1f, 0f, 0f, 0f,
        0f, 1f, 0f, 0f,
        0f, 0f, 1f, 0f,
        offsetX, 0f, 0f, 1f,
    )

Deliberately asymmetric, the way a real HMD's frustum is; a symmetric one would pass even with left and right confused.

    private const val TAN_LEFT = -1.0f
    private const val TAN_RIGHT = 0.8f
    private const val TAN_DOWN = -0.9f
    private const val TAN_UP = 1.1f

A projection carrying those tangents, in a reverse-Z convention the extraction must ignore.

    private val projection = run {
        val m00 = 2f / (TAN_RIGHT - TAN_LEFT)
        val m20 = (TAN_RIGHT + TAN_LEFT) / (TAN_RIGHT - TAN_LEFT)
        val m11 = 2f / (TAN_UP - TAN_DOWN)
        val m21 = (TAN_UP + TAN_DOWN) / (TAN_UP - TAN_DOWN)
        floatArrayOf(
            m00, 0f, 0f, 0f,
            0f, m11, 0f, 0f,
            m20, m21, 0f, -1f,
            0f, 0f, 0.05f, 0f,
        )
    }

fun main(args: Array<String>)

body — 126 lines, from line 79
    @JvmStatic
    fun main(args: Array<String>) {
        installVisionPoseHandler()
        val handler = visionPoseRawHandler()
            ?: error("installVisionPoseHandler registered nothing on $VISION_POSE_CHANNEL")

        VisionPoseStream.reset()
        var seen = 0
        VisionPoseStream.onPose { seen++ }

        // ---- a head announcing itself -----------------------------------------------------------
        require(handler(hello()) == Handled.yes) { "a hello must be handled" }
        val head = VisionPoseStream.heads["test-head"] ?: error("hello did not register the head")
        require(head.viewCount == 2 && head.refreshHz == 90.0) { "hello fields lost: $head" }

        // ---- one pose ---------------------------------------------------------------------------
        require(handler(poseJson(frameIndex = 1)) == Handled.yes) { "a pose frame must be handled" }
        val frame = VisionPoseStream.latest ?: error("a handled pose frame did not become latest")
        require(seen == 1) { "the listener saw $seen poses, expected 1" }
        require(frame.device == "test-head" && frame.frameIndex == 1L) { "identity lost: $frame" }
        require(frame.trackingValid) { "trackingValid lost" }
        require(frame.predictedDisplayTime == 123456.789) { "predictedDisplayTime lost" }
        require(frame.sentAt == 123456.778) { "sentAt lost" }
        require(frame.receivedAtNanos != 0L) { "arrival was not stamped" }

        // The head's own pose, straight off the transform's fourth column.
        assertVec("head position", frame.headPosition, 1f, 1.5f, -2f)
        // 90° about +Y is (0, sin45, 0, cos45).
        val q = frame.headOrientation
        assertClose("head qx", q.x, 0f); assertClose("head qy", q.y, 0.70710678f)
        assertClose("head qz", q.z, 0f); assertClose("head qw", q.w, 0.70710678f)

        require(frame.eyes.size == 2) { "expected two eyes, got ${frame.eyes.size}" }

        // A left eye 32 mm along the *head's* -X, which a 90° yaw points at world +Z: the eye must
        // land at z = -1.968, not at x = 0.968. Getting this wrong is the classic symptom where the
        // stereo pair looks fine standing still and shears when you turn your head.
        val left = frame.eyes[0]
        assertVec("left eye position", Matrix4f(left.viewMatrix).invert().getTranslation(),
                  1f, 1.5f, -1.968f)
        assertVec("right eye position", Matrix4f(frame.eyes[1].viewMatrix).invert().getTranslation(),
                  1f, 1.5f, -2.032f)

        // And the view matrix must actually be world→eye: the eye's own world position maps to zero.
        val leftEyeWorld = Matrix4f(left.viewMatrix).invert().getTranslation()
        assertVec("world→eye origin", left.viewMatrix * leftEyeWorld, 0f, 0f, 0f)

        // Tangents recovered from the projection, depth convention ignored.
        assertClose("tanLeft", left.tanLeft, TAN_LEFT)
        assertClose("tanRight", left.tanRight, TAN_RIGHT)
        assertClose("tanDown", left.tanDown, TAN_DOWN)
        assertClose("tanUp", left.tanUp, TAN_UP)
        // The compositor's own matrix rides through untouched, for anything that wants it verbatim.
        assertClose("projection m32 carried", left.projection.m32, 0.05f)

        // ---- statistics -------------------------------------------------------------------------
        require(handler(poseJson(frameIndex = 2)) == Handled.yes) { "a second pose must be handled" }
        require(VisionPoseStream.frameCount == 2L) { "frameCount = ${VisionPoseStream.frameCount}" }
        require(VisionPoseStream.regressedCount == 0L) { "an advancing frameIndex is not a regression" }
        require(VisionPoseStream.intervalMillis > 0.0) { "arrival interval was never measured" }

        require(handler(poseJson(frameIndex = 2)) == Handled.yes) { "a repeat must still be handled" }
        require(VisionPoseStream.regressedCount == 1L) {
            "a repeated frameIndex must count as a regression, got ${VisionPoseStream.regressedCount}"
        }

        // Latest-wins has to survive out-of-order arrival, because it happens: Field's /ws reader
        // launches a coroutine per raw message, so two poses decode concurrently and finish in
        // either order. Measured at 25 in 968 against the simulator. An older pose applied over a
        // newer one is the "world swims when I turn my head" failure — invisible in a screenshot and
        // not what anyone would look for in the renderer.
        require(handler(poseJson(frameIndex = 1)) == Handled.yes) { "a late pose is still understood" }
        require(VisionPoseStream.latest?.frameIndex == 2L) {
            "a late pose overwrote a newer one: latest is ${VisionPoseStream.latest?.frameIndex}"
        }
        require(VisionPoseStream.regressedCount == 2L) { "the late pose was not counted" }

        // A renderer restart counts from zero again, and treating that as permanently stale would
        // wedge the stream in silence. Under the same device id the only signal is the size of the
        // jump, so this has to be a *large* step back — a two-frame one is reordering, and asserting
        // on that would be asserting the rule away.
        require(handler(poseJson(frameIndex = 1000)) == Handled.yes) { "a normal advance" }
        require(VisionPoseStream.latest?.frameIndex == 1000L) { "frame 1000 did not land" }
        require(handler(poseJson(frameIndex = 0)) == Handled.yes) { "a restart must be handled" }
        require(VisionPoseStream.latest?.frameIndex == 0L) {
            "a restarted head never recovered: latest is ${VisionPoseStream.latest?.frameIndex}"
        }
        // A short step back, at the same distance from a *low* index, is still reordering.
        require(handler(poseJson(frameIndex = 5)) == Handled.yes) { "frame 5 advances" }
        require(handler(poseJson(frameIndex = 4)) == Handled.yes) { "frame 4 is still understood" }
        require(VisionPoseStream.latest?.frameIndex == 5L) { "a one-frame step back was applied" }

        // ---- what must be refused ---------------------------------------------------------------
        val rejectedBefore = VisionPoseStream.rejectedCount
        val stale = poseJson(frameIndex = 9).replace(
            "\"version\":$VISION_POSE_VERSION", "\"version\":${VISION_POSE_VERSION + 1}")
        require(handler(stale) == Handled.no) { "a future wire version must not be handled" }
        require(handler("""{"channel":"$VISION_POSE_CHANNEL", not json""") == Handled.no) {
            "malformed JSON must not be handled"
        }
        require(handler("""{"channel":"$VISION_POSE_CHANNEL","type":"visionpose/frame",""" +
                        """"version":$VISION_POSE_VERSION,"device":"x"}""") == Handled.no) {
            "a frame missing its matrices must not be handled"
        }
        require(VisionPoseStream.latest?.frameIndex == 5L) { "a refused message changed the pose" }
        require(VisionPoseStream.rejectedCount == rejectedBefore + 3) {
            "expected three rejections, got ${VisionPoseStream.rejectedCount - rejectedBefore}"
        }
        // Another channel's traffic is not ours to reject — it must fall through unclaimed. Same for
        // anything that never names the channel: the handler is offered every unrecognised /ws
        // message and takes a substring shortcut before parsing, so junk it never looked at must not
        // land in our statistics. Pinned because that shortcut is an optimisation, and an
        // optimisation that starts swallowing our own traffic would look exactly like silence.
        require(handler("""{"channel":"__visionmetal__","type":"subscribe-all","version":2}""")
                    == Handled.no) { "another channel's message must fall through" }
        require(handler("{not json at all") == Handled.no) { "unrelated junk must fall through" }
        require(VisionPoseStream.rejectedCount == rejectedBefore + 3) {
            "traffic that never names the channel must not be counted as a rejection"
        }

        // ---- the field names the visionOS mirror encodes -----------------------------------------
        assertWireNamesMatchTheHead()

        println("pose summary: ${VisionPoseStream.summary()}")
        println("VisionPoseUplinkSmokeTest ok")
    }

The Swift PoseWire mirror is hand-written, so nothing but a test stops the two spellings drifting. Encoding the Kotlin types and comparing the key set against the documented format catches a renamed or dropped field on this side; Pose/PoseWire.swift carries the same list.

    private fun assertWireNamesMatchTheHead() {
        val wire = VisionPoseWireFrame(
            device = "test-head", frameIndex = 500L,
            predictedDisplayTime = 1.0, sentAt = 1.0, trackingValid = true,
            originFromDevice = originFromDevice,
            views = listOf(VisionPoseWireView(eyeTransform(-0.032f), projection)),
        )
        val encoded = visionPoseWireJson.encodeToString(VisionPoseWireFrame.serializer(), wire)
        val expected = listOf(
            "channel", "type", "version", "device", "frameIndex",
            "predictedDisplayTime", "sentAt", "trackingValid", "originFromDevice", "views",
            "transform", "projection",
        )
        expected.forEach { name ->
            require(encoded.contains("\"$name\"")) { "the wire lost the '$name' field: $encoded" }
        }
        // And the round trip has to survive our own encoder, not just the hand-written JSON.
        require(VisionPoseStream.accept(encoded)) { "the encoder emitted something the decoder refuses" }
        require(VisionPoseStream.latest?.frameIndex == 500L) { "the round-tripped pose was not published" }
    }

    private fun hello() = """
        {"channel":"$VISION_POSE_CHANNEL","type":"visionpose/hello","version":$VISION_POSE_VERSION,
         "device":"test-head","name":"Smoke","viewCount":2,"refreshHz":90.0}
    """.trimIndent()

Hand-written, so the decoder is checked against the documented format rather than its sibling.

    private fun poseJson(frameIndex: Int) = """
        {"channel":"$VISION_POSE_CHANNEL","type":"visionpose/frame","version":$VISION_POSE_VERSION,
         "device":"test-head","frameIndex":$frameIndex,
         "predictedDisplayTime":123456.789,"sentAt":123456.778,"trackingValid":true,
         "originFromDevice":${originFromDevice.joinToString(",", "[", "]")},
         "views":[{"transform":${eyeTransform(-0.032f).joinToString(",", "[", "]")},
                   "projection":${projection.joinToString(",", "[", "]")}},
                  {"transform":${eyeTransform(0.032f).joinToString(",", "[", "]")},
                   "projection":${projection.joinToString(",", "[", "]")}}]}
    """.trimIndent()

    private fun assertClose(what: String, actual: Float, expected: Float, tolerance: Float = 1e-4f) {
        require(abs(actual - expected) <= tolerance) { "$what = $actual, expected $expected" }
    }

    private fun assertVec(what: String, v: Vector3f, x: Float, y: Float, z: Float) {
        assertClose("$what.x", v.x, x); assertClose("$what.y", v.y, y); assertClose("$what.z", v.z, z)
    }
}