OpenXRHmdTarget.kt
src/jvmMain/kotlin/afield/graphics/vulkan/targets/OpenXRHmdTarget.kt · 2998 lines · cited by Field on Apple Vision Pro
package and imports — 98 lines, from line 1
package afield.graphics.vulkan.targets
import afield.graphics.vulkan.device.vkLayout
import afield.math.linalg.Vector2f
import afield.math.linalg.Vector3f
import afield.math.linalg.Vector4f
import afield.app.protocol.Rect
import afield.graphics.vulkan.CPUSpan
import afield.graphics.vulkan.Renderer
import afield.graphics.vulkan.Target
import afield.graphics.vulkan.device.Attachment
import afield.graphics.vulkan.device.CommandBuffer
import afield.graphics.vulkan.device.Fence
import afield.graphics.vulkan.device.GpuProfiler
import afield.graphics.vulkan.device.Instance
import afield.graphics.vulkan.device.ProfilingOptions
import afield.graphics.render.MeshModule
import afield.graphics.render.MeshModulePointer
import afield.graphics.render.Module
import afield.graphics.render.ModulePointer
import afield.document.context.fromContext
import afield.graphics.render.pointer
import afield.graphics.vulkan.buffers.MissingRefException
import afield.graphics.texture.HasImageView
import afield.graphics.texture.image_layout_transition
import afield.graphics.vulkan.util.Options
import afield.app.diagnostics.logging.lprintln
import afield.ai.learning.*
import afield.ai.retrieval.*
import afield.animation.interpolation.*
import afield.animation.keyframes.*
import afield.animation.motion.*
import afield.app.commands.*
import afield.app.diagnostics.*
import afield.audio.analysis.*
import afield.audio.synth.*
import afield.data.collections.*
import afield.data.formats.*
import afield.data.graph.*
import afield.data.sorting.*
import afield.document.context.*
import afield.document.graph.*
import afield.document.properties.*
import afield.document.workspace.*
import afield.geometry.mesh.*
import afield.geometry.path.*
import afield.geometry.spatial.*
import afield.graphics.color.*
import afield.graphics.interaction.*
import afield.graphics.render.*
import afield.graphics.shader.*
import afield.graphics.text.*
import afield.graphics.vulkan.buffers.*
import afield.io.osc.*
import afield.io.process.*
import afield.lang.lambda.*
import afield.lang.python.*
import afield.lang.remotejs.*
import afield.lang.stacksl.*
import afield.math.autodiff.*
import afield.math.control.*
import afield.math.estimation.*
import afield.math.functions.*
import afield.math.optimization.*
import afield.math.statistics.*
import afield.math.units.*
import afield.media.image.*
import afield.media.video.*
import afield.runtime.execution.*
import afield.runtime.languages.*
import afield.runtime.lifecycle.*
import afield.simulation.agents.*
import afield.simulation.automata.*
import afield.simulation.particles.*
import afield.study.build.*
import afield.study.evidence.*
import afield.study.search.*
import afield.study.source.*
import afield.study.ui.*
import afield.ui.controls.*
import afield.ui.output.*
import afield.ui.visualization.*
import afield.math.linalg.remAssign
import afield.math.linalg.Matrix4f
import afield.math.linalg.Quaternionf
import afield.math.linalg.Rectanglef
import org.lwjgl.openxr.*
import org.lwjgl.openxr.KHRVulkanEnable
import org.lwjgl.openxr.KHRVulkanEnable2
import org.lwjgl.openxr.XR10.*
import org.lwjgl.system.MemoryStack
import org.lwjgl.system.MemoryUtil
import org.lwjgl.vulkan.KHRDynamicRendering
import org.lwjgl.vulkan.VK10.*
import org.lwjgl.vulkan.VK13.*
import afield.math.linalg.*
import org.lwjgl.vulkan.*
import kotlin.math.tan
The OpenXR headset as a render target: OpenXRHmdTarget acquires each eye's swapchain image, renders registered modules into it, and submits the frame with a projection layer.
It uses the OpenXR session that the Vulkan instance creates when VR is enabled, and also reports
head and controller poses. Instead of rendering per eye, it can copy a side-by-side texture (from
SideBySideStereoTarget or a wide RenderToTexture) into the two eyes.
WiVRn's serial for its body trackers: wivrn-<n>, n being 1-based
(server/driver/wivrn_generic_tracker.cpp). The index is the joint-whitelist position.
private val WIVRN_TRACKER_SERIAL = Regex("""wivrn-(\d+)""")
private data class XrEyeSwapchain(
val swapchain: XrSwapchain,
val width: Int,
val height: Int,
val format: Int,
val images: LongArray,
val imageViews: LongArray,
val depthAttachments: Array<Attachment>
) {
val colorSeen: BooleanArray = BooleanArray(images.size)
val depthSeen: BooleanArray = BooleanArray(images.size)
// Preallocated viewport rectangles (avoid per-frame heap allocations).
val fullRect: Rect = Rect(0.0, 0.0, width.toDouble(), height.toDouble())
val rectF: Rectanglef = Rectanglef(0f, 0f, width.toFloat(), height.toFloat())
}
class OpenXRHmdTarget
OpenXR HMD render target.
Minimal goal:
- Acquire per-eye swapchain images
- Clear them (default: purple)
- Optionally render Modules/MeshModules into each eye via dynamic rendering
- Submit to Vulkan graphics queue and xrEndFrame with a projection layer
Side-by-side mirror / postprocessing plan:
- Render your whole stereo view into a single offscreen texture (left eye on the left half,
right eye on the right half) using RenderToTexture.
- Use sideBySideRTTParameters to get recommended dimensions: width = 2*eyeWidth, height = eyeHeight.
- Ensure the RTT color image is created with transfer-src usage (RTTParameters.colorTransferSrc = true).
- Present to the HMD by copying halves into the acquired OpenXR swapchain images.
- Call copySideBySideToEyes once to register the source; during renderFrame we do:
left-half -> left eye, right-half -> right eye via vkCmdCopyImage.
- The OpenXR swapchains are created with TRANSFER_DST usage for this path.
- Show the exact same side-by-side RTT in your desktop window however you like (no need to sample
or blit from the OpenXR swapchain images).
Notes: - The copy path assumes the source RTT is usually in SHADER_READ_ONLY_OPTIMAL (current RenderToTexture default) and temporarily transitions it to TRANSFER_SRC_OPTIMAL for the copy, then back. - Call disableSideBySideCopy to go back to the normal per-eye dynamic-rendering path.
This consumes the OpenXR bootstrap created in afield.graphics.vulkan.device.Instance when Options.enableVR is true.
class OpenXRHmdTarget(
/**
* The renderer whose Vulkan device the OpenXR session and its swapchains are created against,
* on that renderer's queue thread, at the first [renderFrame]. Found in the box graph by
* default. The `XrInstance` itself is older than any device and comes from the bootstrap
* [afield.graphics.vulkan.device.Instance] made at startup, which construction claims here.
*/
val renderer: Renderer = fromContext()
) {
@Volatile
private var sideBySideSource: HasImageView? = null
@Volatile
private var layeredSource: HasImageView? = null
Scratch arrays to avoid per-frame allocations.
private var acquiredIndicesScratch: IntArray = IntArray(0)
Per-eye view/projection state (updated from xrLocateViews)
var currentEyeIndex: Int
Index of the eye currently being rendered inside renderEyesVulkan, or -1 when idle.
var currentEyeIndex: Int = -1
private set
private data class EyeViewState(
var px: Float = 0f,
var py: Float = 0f,
var pz: Float = 0f,
var qx: Float = 0f,
var qy: Float = 0f,
var qz: Float = 0f,
var qw: Float = 1f,
var angleLeft: Float = 0f,
var angleRight: Float = 0f,
var angleUp: Float = 0f,
var angleDown: Float = 0f,
)
private var eyeStates: Array<EyeViewState> = emptyArray()
private var eyeViewMatrices: Array<Matrix4f> = emptyArray()
private var eyeProjMatrices: Array<Matrix4f> = emptyArray()
val hasEyeViews: Boolean
True once the runtime has allocated a complete stereo view/projection pair.
val hasEyeViews: Boolean
get() = eyeViewMatrices.size >= 2 && eyeProjMatrices.size >= 2
fun currentEyeViewMatrix(out: Matrix4f = Matrix4f()): Matrix4f
Copies the current eye's view matrix into out and returns it.
Only valid while rendering (i.e. called from module render callbacks).
fun currentEyeViewMatrix(out: Matrix4f = Matrix4f()): Matrix4f {
val i = currentEyeIndex
check(i in eyeViewMatrices.indices) { "No current eye (currentEyeIndex=$i)" }
return out.set(eyeViewMatrices[i])
}
fun currentEyeProjectionMatrix(out: Matrix4f = Matrix4f()): Matrix4f
Copies the current eye's projection matrix into out and returns it.
Only valid while rendering (i.e. called from module render callbacks).
fun currentEyeProjectionMatrix(out: Matrix4f = Matrix4f()): Matrix4f {
val i = currentEyeIndex
check(i in eyeProjMatrices.indices) { "No current eye (currentEyeIndex=$i)" }
return out.set(eyeProjMatrices[i])
}
fun eyeViewMatrix(eyeIndex: Int, out: Matrix4f = Matrix4f()): Matrix4f
Like currentEyeViewMatrix but for an explicit eye index.
fun eyeViewMatrix(eyeIndex: Int, out: Matrix4f = Matrix4f()): Matrix4f {
check(eyeIndex in eyeViewMatrices.indices) { "Invalid eyeIndex=$eyeIndex" }
return out.set(eyeViewMatrices[eyeIndex])
}
fun eyeProjectionMatrix(eyeIndex: Int, out: Matrix4f = Matrix4f()): Matrix4f
Like currentEyeProjectionMatrix but for an explicit eye index.
fun eyeProjectionMatrix(eyeIndex: Int, out: Matrix4f = Matrix4f()): Matrix4f {
check(eyeIndex in eyeProjMatrices.indices) { "Invalid eyeIndex=$eyeIndex" }
return out.set(eyeProjMatrices[eyeIndex])
}
fun headPose(outPosition: Vec3 = Vec3(), outOrientation: Quat = Quat()): Boolean
Approximate head pose from the currently tracked eye views.
Returns false if views are not initialized yet. The position is the average of all eye positions, and the orientation is taken from the first eye (OpenXR typically reports identical orientation per-eye).
Safe from any thread: each frame publishes the whole pose as one snapshot, so a reader on a timer never sees one frame's position with the next frame's orientation.
fun headPose(outPosition: Vec3 = Vec3(), outOrientation: Quat = Quat()): Boolean {
val pose = headPoseSnapshot ?: return false
outPosition.set(pose[0], pose[1], pose[2])
outOrientation.set(pose[3], pose[4], pose[5], pose[6])
return true
}
px py pz qx qy qz qw, replaced whole once per located frame; null until the first one.
@Volatile
private var headPoseSnapshot: FloatArray? = null
private fun publishHeadPose() {
if (eyeStates.isEmpty()) return
var px = 0f
var py = 0f
var pz = 0f
for (s in eyeStates) {
px += s.px
py += s.py
pz += s.pz
}
val inv = 1f / eyeStates.size.toFloat()
val s0 = eyeStates[0]
headPoseSnapshot = floatArrayOf(px * inv, py * inv, pz * inv, s0.qx, s0.qy, s0.qz, s0.qw)
}
Controller state (basic: grip pose + select/squeeze)
enum class Hand { LEFT, RIGHT }
data class JointPose
data class JointPose(
var position: Vec3 = Vec3(),
var orientation: Quat = Quat(),
var radius: Float = 0f,
var positionValid: Boolean = false,
var orientationValid: Boolean = false,
)
enum class HandJoint
enum class HandJoint {
PALM, WRIST,
THUMB_METACARPAL, THUMB_PROXIMAL, THUMB_DISTAL, THUMB_TIP,
INDEX_METACARPAL, INDEX_PROXIMAL, INDEX_INTERMEDIATE, INDEX_DISTAL, INDEX_TIP,
MIDDLE_METACARPAL, MIDDLE_PROXIMAL, MIDDLE_INTERMEDIATE, MIDDLE_DISTAL, MIDDLE_TIP,
RING_METACARPAL, RING_PROXIMAL, RING_INTERMEDIATE, RING_DISTAL, RING_TIP,
LITTLE_METACARPAL, LITTLE_PROXIMAL, LITTLE_INTERMEDIATE, LITTLE_DISTAL, LITTLE_TIP;
companion object { const val COUNT = 26 }
class HandTrackingState
val joints = Array(HandJoint.COUNT) { JointPose() }
var isActive: Boolean = false
var pinchStrength: Float = 0f
operator fun get(joint: HandJoint): JointPose = joints[joint.ordinal]
enum class BodyJoint
The body joints reachable over WiVRn, in the order its client forwards them.
This is deliberately not the 70-joint XR_FB_body_tracking skeleton. WiVRn's client
whitelists a handful of joints out of the headset's inside-out body tracking and relays only
those (see client/xr/fb_body_tracker.cpp), so this enum is the whole of what the PC side
can see. Enum order matches that whitelist, which is what makes the index-to-joint mapping
in inferBodyJointMapping work.
enum class BodyJoint {
HIPS, CHEST,
LEFT_ARM_LOWER, RIGHT_ARM_LOWER,
LEFT_LOWER_LEG, RIGHT_LOWER_LEG,
LEFT_FOOT, RIGHT_FOOT;
companion object { const val COUNT = 8 }
class BodyTrackingState
val joints = Array(BodyJoint.COUNT) { JointPose() }
var isActive: Boolean = false
True while at least one tracker is reporting a valid pose.
var trackedJoints: Set<BodyJoint>
The joints this WiVRn configuration actually reports — the user's "lower body" and "hip" toggles in the headset decide. Joints outside this set are never written and stay invalid; empty means we could not identify the trackers (see bodyTrackerPose).
var trackedJoints: Set<BodyJoint> = emptySet()
internal set
operator fun get(joint: BodyJoint): JointPose = joints[joint.ordinal]
data class ControllerState
body — 76 lines, from line 342
data class ControllerState(
var active: Boolean = false,
var poseValid: Boolean = false,
var position : Vec3 = Vec3(),
var rotation : Quat = Quat(),
// Generic select (bound per-profile; for Index we bind trigger/click)
var selectPressed: Boolean = false,
// Trigger
var triggerValue: Float = 0f,
var triggerPressed: Boolean = false,
// Grip / squeeze
var squeezeValue: Float = 0f,
var squeezePressed: Boolean = false,
// Trackpad (Vive + Index)
var trackpadX: Float = 0f,
var trackpadY: Float = 0f,
var trackpadTouched: Boolean = false,
var trackpadPressed: Boolean = false,
// Thumbstick (Index)
var thumbstickX: Float = 0f,
var thumbstickY: Float = 0f,
var thumbstickTouched: Boolean = false,
var thumbstickPressed: Boolean = false,
// Buttons (Index)
var aPressed: Boolean = false,
var bPressed: Boolean = false,
// Menu (Vive)
var menuPressed: Boolean = false,
)
private val leftController = ControllerState()
private val rightController = ControllerState()
private var xrActionSet: XrActionSet? = null
private var xrPoseAction: XrAction? = null
private var xrAimPoseAction: XrAction? = null
private var xrSelectAction: XrAction? = null
private var xrTriggerValueAction: XrAction? = null
private var xrTriggerClickAction: XrAction? = null
private var xrSqueezeValueAction: XrAction? = null
private var xrSqueezeClickAction: XrAction? = null
private var xrTrackpadAction: XrAction? = null
private var xrTrackpadTouchAction: XrAction? = null
private var xrTrackpadClickAction: XrAction? = null
private var xrThumbstickAction: XrAction? = null
private var xrThumbstickTouchAction: XrAction? = null
private var xrThumbstickClickAction: XrAction? = null
private var xrAAction: XrAction? = null
private var xrBAction: XrAction? = null
private var xrMenuAction: XrAction? = null
private var xrHapticAction: XrAction? = null
private var leftHandPath: Long = XR_NULL_PATH
private var rightHandPath: Long = XR_NULL_PATH
private var leftGripSpace: XrSpace? = null
private var rightGripSpace: XrSpace? = null
private var leftAimSpace: XrSpace? = null
private var rightAimSpace: XrSpace? = null
private var handTrackingEnabled = false
private var leftHandTracker: XrHandTrackerEXT? = null
private var rightHandTracker: XrHandTrackerEXT? = null
private var leftJointsBuf: XrHandJointLocationEXT.Buffer? = XrHandJointLocationEXT.calloc(HandJoint.COUNT)
private var rightJointsBuf: XrHandJointLocationEXT.Buffer? = XrHandJointLocationEXT.calloc(HandJoint.COUNT)
private val leftHandState = HandTrackingState()
private val rightHandState = HandTrackingState()
private var eyeGazeInteraction: ExtEyeGazeInteraction? = null
private var htcEyeTracker: HtcEyeTracker? = null
fun eyeGazeState(): EyeGazeState? = eyeGazeInteraction?.state
Combined gaze from XR_EXT_eye_gaze_interaction, or null when unavailable/not initialized. The state is updated in place; check poseValid (and isTracked for nominal-quality gaze).
fun htcEyeState(): HtcEyeTrackingState? = htcEyeTracker?.state
Per-eye gaze, pupil and geometry from XR_HTC_eye_tracker, or null when unavailable. Independent of eyeGazeState; check each observation's validity before using its values.
One WiVRn generic tracker: its whitelist index, its space, and the pose we located.
private class BodyTracker(val index: Int, val space: XrSpace) {
val pose = JointPose()
}
private var xdevSpaceExt: MndxXDevSpace? = null
private var xdevList: Long = 0L
private var xdevGeneration: Long = -1L
private var bodyTrackingEnabled = false
private var bodyTrackers: List<BodyTracker> = emptyList()
private var bodyJointMapping: List<BodyJoint> = emptyList()
private val bodyTrackingState = BodyTrackingState()
private var inputDebugPrinted = false
fun handState(hand: Hand): HandTrackingState?
fun handState(hand: Hand): HandTrackingState? {
if (!handTrackingEnabled) return null
return if (hand == Hand.LEFT) leftHandState else rightHandState
}
fun bodyState(): BodyTrackingState?
Null when the runtime exposes no body trackers at all.
fun bodyState(): BodyTrackingState? {
if (!bodyTrackingEnabled) return null
return bodyTrackingState
}
fun bodyTrackerCount(): Int = bodyTrackers.size
How many raw body trackers the runtime is publishing.
fun bodyTrackerPose(index: Int): JointPose? = bodyTrackers.getOrNull(index)?.pose
The raw pose of the n-th tracker, bypassing the semantic mapping. Use this when the joint mapping came out empty — a runtime other than WiVRn, or a tracker count we don't recognise — and you would still rather have the poses than nothing.
fun controllerState(hand: Hand, out: ControllerState = ControllerState()): ControllerState
fun controllerState(hand: Hand, out: ControllerState = ControllerState()): ControllerState {
val src = if (hand == Hand.LEFT) leftController else rightController
out.active = src.active
out.poseValid = src.poseValid
out.position %= src.position
out.rotation %= src.rotation
out.selectPressed = src.selectPressed
out.triggerValue = src.triggerValue
out.triggerPressed = src.triggerPressed
out.squeezeValue = src.squeezeValue
out.squeezePressed = src.squeezePressed
out.trackpadX = src.trackpadX
out.trackpadY = src.trackpadY
out.trackpadTouched = src.trackpadTouched
out.trackpadPressed = src.trackpadPressed
out.thumbstickX = src.thumbstickX
out.thumbstickY = src.thumbstickY
out.thumbstickTouched = src.thumbstickTouched
out.thumbstickPressed = src.thumbstickPressed
out.aPressed = src.aPressed
out.bPressed = src.bPressed
out.menuPressed = src.menuPressed
return out
}
fun hapticPulse
fun hapticPulse(
hand: Hand,
amplitude: Float = 0.5f,
durationNs: Long = 20_000_000L,
frequencyHz: Float = 0f,
) {
val session = xrSession ?: return
val hapticAction = xrHapticAction ?: return
val subactionPath = if (hand == Hand.LEFT) leftHandPath else rightHandPath
MemoryStack.stackPush().use { stack ->
val vib = XrHapticVibration.calloc(stack)
.type(XR_TYPE_HAPTIC_VIBRATION)
.amplitude(amplitude)
.duration(durationNs)
.frequency(frequencyHz)
val hapticBase = XrHapticBaseHeader.create(vib.address())
val info = XrHapticActionInfo.calloc(stack)
.type(XR_TYPE_HAPTIC_ACTION_INFO)
.action(hapticAction)
.subactionPath(subactionPath)
xrApplyHapticFeedback(session, info, hapticBase)
}
}
fun controllerGripPoseMatrix(hand: Hand, out: Matrix4f = Matrix4f()): Matrix4f
fun controllerGripPoseMatrix(hand: Hand, out: Matrix4f = Matrix4f()): Matrix4f {
val s = if (hand == Hand.LEFT) leftController else rightController
return out.identity()
.translate(s.position.x, s.position.y, s.position.z)
.rotate(Quaternionf(s.rotation.x, s.rotation.y, s.rotation.z, s.rotation.w))
}
var modules = mutableMapOf<String, ModulePointer>()
The graphics modules to draw, by name, the same way RenderTarget holds them. Every module
here is rendered once per eye; adding a name that is already present replaces it.
var meshModules = mutableMapOf<String, MeshModulePointer>()
The mesh-shader modules to draw, by name, rendered per eye alongside modules.
var clearColor: FloatArray = Options.openXRClearColor.clone()
What each eye's swapchain image is cleared to, as RGBA, copied from Options.openXRClearColor
at construction. An array of fewer than four is read channel by channel, with the missing
channels taken from opaque magenta (1, 0, 1, 1) rather than from zero.
fun setClearColor(r: Float, g: Float, b: Float, a: Float = 1f)
fun setClearColor(r: Float, g: Float, b: Float, a: Float = 1f) {
clearColor = floatArrayOf(r, g, b, a)
}
fun sideBySideRTTParameters(parameters: RTTParameters.() -> Unit
Convenience: Build an RTTParameters for a single side-by-side render target (left eye on the left half, right eye on the right half) matching the OpenXR recommended eye dimensions.
Requires this target to have initialized its swapchains (i.e. after the first renderFrame call).
fun sideBySideRTTParameters(parameters: RTTParameters.() -> Unit = {}): RTTParameters {
check(swapchains.size >= 2) {
"OpenXR swapchains not initialized yet; call renderFrame() once before requesting RTT parameters"
}
val left = swapchains[0]
val right = swapchains[1]
check(left.width == right.width && left.height == right.height) {
"Expected stereo eyes to share the same size; got L=${left.width}x${left.height}, R=${right.width}x${right.height}"
}
return RTTParameters(
width = left.width * 2,
height = left.height,
mipLevels = 1,
format = left.format,
colorAttachments = 1,
colorFormats = listOf(left.format),
colorLayerCount = 1,
colorSampled = true,
colorTransferSrc = true,
useDepth = true,
).apply(parameters)
}
fun copySideBySideToEyes(source: HasImageView)
Convenience: Copy a side-by-side image into the OpenXR eyes every frame.
The source is interpreted as: - left half (x in [0, eyeWidth)) -> left eye - right half (x in [eyeWidth, 2*eyeWidth)) -> right eye
This is executed during renderFrame (after acquiring swapchain images, before releasing them).
fun copySideBySideToEyes(source: HasImageView) {
sideBySideSource = source
layeredSource = null
}
fun copyLayersToEyes(source: HasImageView)
Copy array layer zero/one directly into the acquired left/right OpenXR images.
fun copyLayersToEyes(source: HasImageView) {
require(source.imageView.imageViewData.layerCount >= 2) { "Stereo source needs two array layers" }
layeredSource = source
sideBySideSource = null
}
fun clearEyeSource(source: HasImageView)
fun clearEyeSource(source: HasImageView) {
if (layeredSource === source) layeredSource = null
if (sideBySideSource === source) sideBySideSource = null
}
fun disableSideBySideCopy()
fun disableSideBySideCopy() {
sideBySideSource = null
}
operator fun plusAssign(m: Pair<String, ModulePointer>)
@JvmName("plusAssign")
operator fun plusAssign(m: Pair<String, ModulePointer>) {
if (m.first in modules) modules.remove(m.first)
modules[m.first] = m.second
}
operator fun plusAssign(m: ModulePointer)
@JvmName("plusAssign_mp")
operator fun plusAssign(m: ModulePointer) {
modules[m.current.parameters.name] = m
}
operator fun plusAssign(m: Pair<String, Module>)
@JvmName("plusAssign_module")
operator fun plusAssign(m: Pair<String, Module>) {
modules[m.first] = m.second.pointer()
}
operator fun plusAssign(m: Pair<String, MeshModulePointer>)
@JvmName("plusAssign_mesh")
operator fun plusAssign(m: Pair<String, MeshModulePointer>) {
meshModules[m.first] = m.second
}
operator fun plusAssign(m: MeshModulePointer)
@JvmName("plusAssign_mmp")
operator fun plusAssign(m: MeshModulePointer) {
meshModules[m.current.parameters.name] = m
}
operator fun plusAssign(m: Pair<String, MeshModule>)
@JvmName("plusAssign_mmodule")
operator fun plusAssign(m: Pair<String, MeshModule>) {
meshModules[m.first] = m.second.pointer()
}
OpenXR handles
private var xrBootstrap: Instance.Companion.OpenXRBootstrap? = null
private var xrInstance: XrInstance? = null
private var xrSystemId: Long = 0
private var xrSession: XrSession? = null
private var xrSpace: XrSpace? = null
private var viewCount: Int = 0
private var viewConfigs: XrViewConfigurationView.Buffer? = null
private val swapchains = mutableListOf<XrEyeSwapchain>()
state
private var sessionRunning = false
private var sessionState = XR_SESSION_STATE_UNKNOWN
Vulkan submission resources
private val cmd = CommandBuffer(renderer.commandPool, primary = true, oneTimeSubmit = true)
private val fence = Fence(renderer.device, signaled = true)
lazy init guard
private var initialized = false
private var disabledReason: String? = null
data class XrFrameMarkers
Profiling infrastructure (similar to Renderer)
data class XrFrameMarkers(
var tsFrameBegin: Int = -1,
var tsAfterEyes: Int = -1,
var tsFrameEnd: Int = -1,
var pipeStatsEyes: Int = -1,
var perfFrameIndex: Int = -1
)
data class XrEyeMarks
data class XrEyeMarks(
val eyeIndex: Int,
var tsBegin: Int = -1,
var tsEnd: Int = -1,
var pipeStatsIndex: Int = -1,
var perfIndex: Int = -1
)
data class XrProfilerState
data class XrProfilerState(
val profiler: GpuProfiler,
var markers: XrFrameMarkers = XrFrameMarkers(),
var perFrameEyeMarks: MutableList<XrEyeMarks> = mutableListOf(),
val pending: ArrayDeque<Pending> = ArrayDeque(),
val lock: Any = Any()
)
data class Pending(val frameCounter: Long)
private companion object { const val READBACK_LAG = 3 }
private var profilerState: XrProfilerState? = null
var xrFrameCounter
How many OpenXR frames this target has worked through, counted at the end of renderFrame's
queue-thread pass rather than by the renderer, so it does not follow Runloop.frame and does
not advance while the target is disabled. It is the frame identity the GPU profiler pairs its
readbacks with, and what the XR CPU-span report prints against.
var xrFrameCounter = 0L
private set
init
init {
run {
if (!Options.enableVR) {
disabledReason = "Options.enableVR is false"
return@run
}
// Consume the OpenXR bootstrap from Instance (created before VkInstance/VkDevice).
// Ownership of the XrInstance transfers to this target.
val b = Instance.consumeOpenXRBootstrap()
if (b == null) {
disabledReason = "OpenXR bootstrap missing (Instance.consumeOpenXRBootstrap() returned null)"
return@run
}
xrBootstrap = b
xrInstance = b.xrInstance
xrSystemId = b.systemId
// full initialization happens on first renderFrame() so construction stays cheap
}
}
fun renderFrame(renderScene: () -> Unit
Wait and locate this frame before renderScene records desktop/stereo work, then submit
OpenXR using exactly those views and that predicted display time. The scene callback runs
on the caller thread, outside the queue executor, so graphics workers can submit safely.
It also runs when XR is inactive, keeping the desktop usable while a headset is asleep.
fun renderFrame(renderScene: () -> Unit = {}) {
withOpenXRFrame(::prepareFrame, renderScene, ::submitFrame) { it.views.free() }
}
private class PreparedFrame(
val session: XrSession,
val space: XrSpace,
val displayTime: Long,
val views: XrView.Buffer,
var viewsLocated: Boolean = false,
)
var locatedViewFrameId: Long
Frame whose views scene render callbacks currently read; -1 before the first locate.
var locatedViewFrameId: Long = -1L
private set
var locatedViewDisplayTime: Long
Predicted display time, in the runtime's clock, shared by scene rendering and submission.
body — 974 lines, from line 746
var locatedViewDisplayTime: Long = 0L
private set
private fun prepareFrame(): PreparedFrame? {
if (disabledReason != null) return null
if (!initialized) {
renderer.runOnQueueThread { initSessionAndSwapchains() }
initialized = true
}
// Drain ready frames (no stalls) for profiling readback
profilerState?.let { ps ->
synchronized(ps.lock) {
while (ps.pending.isNotEmpty()) {
val head = ps.pending.first()
val enoughLag = ps.pending.size > READBACK_LAG
// For XR we only check lag, not fence (fence is per-frame, not per-swapchain-image)
if (!enoughLag) break
resolveProfilingForFrame(head.frameCounter)
ps.pending.removeFirst()
}
}
}
return renderer.runOnQueueThread {
// Fail closed even when events, frame waiting, view location or rendering are skipped.
eyeGazeInteraction?.state?.invalidate()
htcEyeTracker?.state?.invalidate()
CPUSpan.time("xr_events") {
if (!pollEvents()) return@runOnQueueThread null
}
if (!sessionRunning) return@runOnQueueThread null
val xrSessionLocal = xrSession ?: return@runOnQueueThread null
val xrSpaceLocal = xrSpace ?: return@runOnQueueThread null
MemoryStack.stackPush().use { stack ->
// Wait for frame
val frameWaitInfo = XrFrameWaitInfo.calloc(stack).type(XR_TYPE_FRAME_WAIT_INFO)
val frameState = XrFrameState.calloc(stack).type(XR_TYPE_FRAME_STATE)
var res = XR_SUCCESS
CPUSpan.time("xr_waitFrame") {
res = xrWaitFrame(xrSessionLocal, frameWaitInfo, frameState)
}
if (res != XR_SUCCESS) return@use null
// Begin profiling for this frame
profilerState?.let { ps ->
synchronized(ps.lock) {
ps.profiler.beginFrame(0) // single-frame index
ps.markers = XrFrameMarkers()
ps.perFrameEyeMarks.clear()
}
}
// Begin frame
val frameBeginInfo = XrFrameBeginInfo.calloc(stack).type(XR_TYPE_FRAME_BEGIN_INFO)
CPUSpan.time("xr_beginFrame") {
res = xrBeginFrame(xrSessionLocal, frameBeginInfo)
}
if (res < 0) return@use null
val frame = PreparedFrame(xrSessionLocal, xrSpaceLocal, frameState.predictedDisplayTime(), XrView.calloc(viewCount))
for (i in 0 until viewCount) frame.views[i].type(XR_TYPE_VIEW)
try {
if (frameState.shouldRender()) {
// Locate views
val viewLocateInfo = XrViewLocateInfo.calloc(stack)
.type(XR_TYPE_VIEW_LOCATE_INFO)
.viewConfigurationType(XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO)
.displayTime(frameState.predictedDisplayTime())
.space(xrSpaceLocal)
val viewState = XrViewState.calloc(stack).type(XR_TYPE_VIEW_STATE)
val viewCapacityBuf = stack.ints(viewCount)
val viewsBuffer = frame.views
CPUSpan.time("xr_locateViews") {
res = xrLocateViews(xrSessionLocal, viewLocateInfo, viewState, viewCapacityBuf, viewsBuffer)
}
if (res == XR_SUCCESS) {
updateEyeMatricesFromLocatedViews(viewsBuffer)
CPUSpan.time("xr_controllers") {
syncAndUpdateControllers(xrSessionLocal, xrSpaceLocal, frameState.predictedDisplayTime())
}
if (handTrackingEnabled) CPUSpan.time("xr_hand_tracking") {
syncHandTracking(xrSessionLocal, xrSpaceLocal, frameState.predictedDisplayTime())
}
if (xdevList != 0L) CPUSpan.time("xr_body_tracking") {
syncBodyTracking(xrSessionLocal, xrSpaceLocal, frameState.predictedDisplayTime())
}
if (htcEyeTracker != null && sessionState == XR_SESSION_STATE_FOCUSED) CPUSpan.time("xr_htc_eye_tracking") {
htcEyeTracker?.update(xrSpaceLocal, frameState.predictedDisplayTime())
}
frame.viewsLocated = true
locatedViewFrameId = xrFrameCounter
locatedViewDisplayTime = frame.displayTime
}
}
frame
} catch (failure: Throwable) {
try { submitFrame(frame, false) }
catch (endFailure: Throwable) { failure.addSuppressed(endFailure) }
finally { frame.views.free() }
throw failure
}
}
}
}
private fun submitFrame(frame: PreparedFrame, sceneRendered: Boolean) {
renderer.runOnQueueThread {
MemoryStack.stackPush().use { stack ->
var layerCount = 0
val layers = stack.callocPointer(1)
try {
if (sceneRendered && frame.viewsLocated) {
// Acquire all swapchain images first (CPU), then render, then submit+wait, then release.
if (acquiredIndicesScratch.size != swapchains.size) {
acquiredIndicesScratch = IntArray(swapchains.size)
}
val acquiredIndices = acquiredIndicesScratch
java.util.Arrays.fill(acquiredIndices, -1)
CPUSpan.time("xr_acquire") {
for (i in swapchains.indices) {
val sc = swapchains[i]
val acquireInfo = XrSwapchainImageAcquireInfo.calloc(stack)
.type(XR_TYPE_SWAPCHAIN_IMAGE_ACQUIRE_INFO)
val idxBuf = stack.callocInt(1)
xrAcquireSwapchainImage(sc.swapchain, acquireInfo, idxBuf)
val imageIndex = idxBuf[0]
acquiredIndices[i] = imageIndex
val waitInfo = XrSwapchainImageWaitInfo.calloc(stack)
.type(XR_TYPE_SWAPCHAIN_IMAGE_WAIT_INFO)
.timeout(XR_INFINITE_DURATION)
xrWaitSwapchainImage(sc.swapchain, waitInfo)
}
}
// Record + submit Vulkan work that renders into the acquired swapchain images.
// NOTE: Depending on Options.openXRWaitForGpuBeforeRelease, we may or may not
// hard-wait for GPU completion before releasing.
CPUSpan.time("xr_render") {
renderEyesVulkan(acquiredIndices)
}
if (Options.openXRWaitForGpuBeforeRelease) {
CPUSpan.time("xr_fence_wait") {
fence.fenceWait()
}
}
// Release swapchain images.
CPUSpan.time("xr_release") {
for (i in swapchains.indices) {
val sc = swapchains[i]
val releaseInfo = XrSwapchainImageReleaseInfo.calloc(stack)
.type(XR_TYPE_SWAPCHAIN_IMAGE_RELEASE_INFO)
xrReleaseSwapchainImage(sc.swapchain, releaseInfo)
}
}
// Build projection views
val projectionViews = XrCompositionLayerProjectionView.calloc(swapchains.size, stack)
for (i in swapchains.indices) {
val sc = swapchains[i]
val view = frame.views[i]
projectionViews[i]
.type(XR_TYPE_COMPOSITION_LAYER_PROJECTION_VIEW)
.pose(view.pose())
.fov(view.fov())
projectionViews[i].subImage()
.swapchain(sc.swapchain)
.imageArrayIndex(0)
projectionViews[i].subImage().imageRect().offset().x(0).y(0)
projectionViews[i].subImage().imageRect().extent()
.width(sc.width)
.height(sc.height)
}
val projectionLayer = XrCompositionLayerProjection.calloc(stack)
.type(XR_TYPE_COMPOSITION_LAYER_PROJECTION)
.space(frame.space)
.views(projectionViews)
layers.put(0, projectionLayer)
layerCount = 1
}
} finally {
// Even a failed scene/render must balance xrBeginFrame. Submit no layer on
// failure instead of leaving the runtime waiting for an abandoned frame.
val endInfo = XrFrameEndInfo.calloc(stack)
.type(XR_TYPE_FRAME_END_INFO)
.displayTime(frame.displayTime)
.environmentBlendMode(XR_ENVIRONMENT_BLEND_MODE_OPAQUE)
.layerCount(layerCount)
.layers(if (layerCount > 0) layers else null)
CPUSpan.time("xr_endFrame") { xrEndFrame(frame.session, endInfo) }
profilerState?.pending?.addLast(Pending(xrFrameCounter))
xrFrameCounter++
}
}
CPUSpan.snapshotAndMaybePrint(everyNFrames = 120, frame = xrFrameCounter, header = "XR CPU")
}
}
private fun initSessionAndSwapchains() {
val xrInstanceLocal = xrInstance ?: return
MemoryStack.stackPush().use { stack ->
// Query view configuration
val viewCountBuf = stack.callocInt(1)
xrEnumerateViewConfigurationViews(
xrInstanceLocal,
xrSystemId,
XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO,
viewCountBuf,
null
)
viewCount = viewCountBuf[0]
eyeStates = Array(viewCount) { EyeViewState() }
eyeViewMatrices = Array(viewCount) { Matrix4f() }
eyeProjMatrices = Array(viewCount) { Matrix4f() }
viewConfigs = XrViewConfigurationView.calloc(viewCount)
for (i in 0 until viewCount) {
viewConfigs!![i].type(XR_TYPE_VIEW_CONFIGURATION_VIEW)
}
xrEnumerateViewConfigurationViews(
xrInstanceLocal,
xrSystemId,
XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO,
viewCountBuf,
viewConfigs
)
// Create XR session
val graphicsBinding = XrGraphicsBindingVulkan2KHR.calloc(stack)
// LWJGL quirk: the type constant for the binding struct is XR_TYPE_GRAPHICS_BINDING_VULKAN_KHR
.type(KHRVulkanEnable.XR_TYPE_GRAPHICS_BINDING_VULKAN_KHR)
.instance(renderer.instance.vkInstance)
.physicalDevice(renderer.physicalDevice.vkPhysicalDevice)
.device(renderer.device.vkDevice)
.queueFamilyIndex(renderer.graphQueue.queueFamilyIndex)
.queueIndex(0)
val sessionCreateInfo = XrSessionCreateInfo.calloc(stack)
.type(XR_TYPE_SESSION_CREATE_INFO)
.next(graphicsBinding.address())
.systemId(xrSystemId)
val sessionPtr = stack.callocPointer(1)
val res = xrCreateSession(xrInstanceLocal, sessionCreateInfo, sessionPtr)
check(res == XR_SUCCESS) { "xrCreateSession failed: $res" }
val session = XrSession(sessionPtr[0], xrInstanceLocal)
xrSession = session
// Reference space
val spaceCreateInfo = XrReferenceSpaceCreateInfo.calloc(stack)
.type(XR_TYPE_REFERENCE_SPACE_CREATE_INFO)
.referenceSpaceType(XR_REFERENCE_SPACE_TYPE_LOCAL)
spaceCreateInfo.poseInReferenceSpace().orientation().w(1f)
val spacePtr = stack.callocPointer(1)
val spRes = xrCreateReferenceSpace(session, spaceCreateInfo, spacePtr)
check(spRes == XR_SUCCESS) { "xrCreateReferenceSpace failed: $spRes" }
xrSpace = XrSpace(spacePtr[0], session)
initControllerActions(session)
initHandTracking(session)
initBodyTracking(session)
if (HtcEyeTracker.EXTENSION_NAME in Instance.enabledOpenXRInstanceExtensions) {
htcEyeTracker = HtcEyeTracker.create(session, xrSystemId)
}
// Swapchain format selection (prefer SRGB)
val formatCountBuf = stack.callocInt(1)
xrEnumerateSwapchainFormats(session, formatCountBuf, null)
val formatCount = formatCountBuf[0]
check(formatCount > 0) { "No OpenXR swapchain formats" }
val formats = stack.callocLong(formatCount)
xrEnumerateSwapchainFormats(session, formatCountBuf, formats)
val formatList = IntArray(formatCount) { formats[it].toInt() }
val preferred = intArrayOf(
VK_FORMAT_B8G8R8A8_SRGB,
VK_FORMAT_R8G8B8A8_SRGB,
VK_FORMAT_B8G8R8A8_UNORM,
VK_FORMAT_R8G8B8A8_UNORM
)
val selectedFormat = preferred.firstOrNull { pf -> formatList.any { it == pf } } ?: formatList[0]
// Create per-eye swapchains
swapchains.clear()
for (i in 0 until viewCount) {
val cfg = viewConfigs!![i]
val w = cfg.recommendedImageRectWidth()
val h = cfg.recommendedImageRectHeight()
val samples = cfg.recommendedSwapchainSampleCount().coerceAtLeast(1)
val sci = XrSwapchainCreateInfo.calloc(stack)
.type(XR_TYPE_SWAPCHAIN_CREATE_INFO)
.usageFlags((XR_SWAPCHAIN_USAGE_COLOR_ATTACHMENT_BIT or XR_SWAPCHAIN_USAGE_TRANSFER_DST_BIT).toLong())
.format(selectedFormat.toLong())
.sampleCount(samples)
.width(w)
.height(h)
.faceCount(1)
.arraySize(1)
.mipCount(1)
val pSwapchain = stack.callocPointer(1)
val scRes = xrCreateSwapchain(session, sci, pSwapchain)
check(scRes == XR_SUCCESS) { "xrCreateSwapchain failed: $scRes" }
val swapchain = XrSwapchain(pSwapchain[0], session)
// Enumerate images
val imageCountBuf = stack.callocInt(1)
val imageCountRes = xrEnumerateSwapchainImages(swapchain, imageCountBuf, null)
check(imageCountRes == XR_SUCCESS) {
"xrEnumerateSwapchainImages(count) failed: $imageCountRes"
}
val imageCount = imageCountBuf[0]
check(imageCount > 0) { "OpenXR swapchain returned no Vulkan images" }
val swapchainImages = XrSwapchainImageVulkan2KHR.calloc(imageCount, stack)
for (j in 0 until imageCount) {
// Like the Vulkan2 graphics binding/requirements structs, LWJGL's Vulkan2
// wrapper is an alias but SteamVR expects the original KHR structure type.
swapchainImages[j].type(KHRVulkanEnable.XR_TYPE_SWAPCHAIN_IMAGE_VULKAN_KHR)
}
val imageRes = xrEnumerateSwapchainImages(
swapchain,
imageCountBuf,
XrSwapchainImageBaseHeader.create(swapchainImages)
)
check(imageRes == XR_SUCCESS) {
"xrEnumerateSwapchainImages(images) failed: $imageRes"
}
val images = LongArray(imageCount) { swapchainImages[it].image() }
// Create VkImageViews and depth attachments for each swapchain image.
val imageViews = LongArray(imageCount)
val depth = Array(imageCount) {
Attachment(
renderer.device,
w,
h,
Options.defaultDepthFormat,
VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT
)
}
for (j in 0 until imageCount) {
imageViews[j] = createImageView(images[j], selectedFormat)
}
swapchains.add(
XrEyeSwapchain(
swapchain = swapchain,
width = w,
height = h,
format = selectedFormat,
images = images,
imageViews = imageViews,
depthAttachments = depth
)
)
}
// Initialize profiling if enabled
if (Options.profiling) {
profilerState = createProfilerState()
}
}
}
private fun createProfilerState(): XrProfilerState {
val opts = ProfilingOptions(
enabled = true,
timestamps = true,
pipelineStats = Options.isWindows,
perfCounters = Options.isWindows
)
// We use a single frame for OpenXR profiling since we don't have per-swapchain-image tracking
val profiler = GpuProfiler(
device = renderer.device.vkDevice,
physicalDevice = renderer.device.physicalDevice.vkPhysicalDevice,
queueFamilyIndex = renderer.graphQueue.queueFamilyIndex,
framesInFlight = 1,
opts = opts
)
return XrProfilerState(profiler)
}
private fun updateEyeMatricesFromLocatedViews(views: XrView.Buffer) {
val nearF = Options.zNear.toFloat()
val farF = Options.zFar.toFloat()
for (i in 0 until viewCount.coerceAtMost(views.capacity())) {
val v = views[i]
val pose = v.pose()
val pos = pose.`position$`()
val ori = pose.orientation()
val fov = v.fov()
val s = eyeStates[i]
s.px = pos.x(); s.py = pos.y(); s.pz = pos.z()
s.qx = ori.x(); s.qy = ori.y(); s.qz = ori.z(); s.qw = ori.w()
s.angleLeft = fov.angleLeft()
s.angleRight = fov.angleRight()
s.angleUp = fov.angleUp()
s.angleDown = fov.angleDown()
computeViewMatrix(s, eyeViewMatrices[i])
computeProjectionMatrix(s, nearF, farF, eyeProjMatrices[i])
}
publishHeadPose()
}
private fun computeViewMatrix(state: EyeViewState, out: Matrix4f) {
// OpenXR view pose is a rigid transform in the chosen reference space.
// View matrix is the inverse of that pose (world->view).
val q = Quaternionf(state.qx, state.qy, state.qz, state.qw)
val p = Vector3f(state.px, state.py, state.pz)
out.identity()
.translate(p)
.rotate(q)
.invert()
}
private fun computeProjectionMatrix(state: EyeViewState, zNear: Float, zFar: Float, out: Matrix4f) {
// Asymmetric frustum from OpenXR fov angles.
val left = tan(state.angleLeft.toDouble()).toFloat() * zNear
val right = tan(state.angleRight.toDouble()).toFloat() * zNear
val bottom = tan(state.angleDown.toDouble()).toFloat() * zNear
val top = tan(state.angleUp.toDouble()).toFloat() * zNear
out.identity().frustum(left, right, bottom, top, zNear, zFar, true)
}
private fun initControllerActions(session: XrSession) {
val xrInstanceLocal = xrInstance ?: return
MemoryStack.stackPush().use { stack ->
// Paths for hands + profile
leftHandPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left")
rightHandPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right")
val profileSimpleController = stringToPath(xrInstanceLocal, stack, "/interaction_profiles/khr/simple_controller")
val profileOculusTouch = stringToPath(xrInstanceLocal, stack, "/interaction_profiles/oculus/touch_controller")
val profileValveIndex = stringToPath(xrInstanceLocal, stack, "/interaction_profiles/valve/index_controller")
val profileHTCVive = stringToPath(xrInstanceLocal, stack, "/interaction_profiles/htc/vive_controller")
val profileMicrosoftMotion = stringToPath(xrInstanceLocal, stack, "/interaction_profiles/microsoft/motion_controller")
// Action set
val actionSetInfo = XrActionSetCreateInfo.calloc(stack)
.type(XR_TYPE_ACTION_SET_CREATE_INFO)
.actionSetName(stack.UTF8("gameplay"))
.localizedActionSetName(stack.UTF8("Gameplay"))
.priority(0)
val pActionSet = stack.callocPointer(1)
val asRes = xrCreateActionSet(xrInstanceLocal, actionSetInfo, pActionSet)
check(asRes == XR_SUCCESS) { "xrCreateActionSet failed: $asRes" }
val actionSet = XrActionSet(pActionSet[0], xrInstanceLocal)
xrActionSet = actionSet
val subactionPaths = stack.longs(leftHandPath, rightHandPath)
// Pose action (grip pose)
val poseInfo = XrActionCreateInfo.calloc(stack)
.type(XR_TYPE_ACTION_CREATE_INFO)
.actionType(XR_ACTION_TYPE_POSE_INPUT)
.actionName(stack.UTF8("grip_pose"))
.localizedActionName(stack.UTF8("Grip Pose"))
.countSubactionPaths(subactionPaths.remaining())
.subactionPaths(subactionPaths)
val pPoseAction = stack.callocPointer(1)
val paRes = xrCreateAction(actionSet, poseInfo, pPoseAction)
check(paRes == XR_SUCCESS) { "xrCreateAction(pose) failed: $paRes" }
val poseAction = XrAction(pPoseAction[0], actionSet)
xrPoseAction = poseAction
// Pose action (aim pose) - many controllers report aim pose more reliably than grip.
val aimInfo = XrActionCreateInfo.calloc(stack)
.type(XR_TYPE_ACTION_CREATE_INFO)
.actionType(XR_ACTION_TYPE_POSE_INPUT)
.actionName(stack.UTF8("aim_pose"))
.localizedActionName(stack.UTF8("Aim Pose"))
.countSubactionPaths(subactionPaths.remaining())
.subactionPaths(subactionPaths)
val pAimAction = stack.callocPointer(1)
val aaRes = xrCreateAction(actionSet, aimInfo, pAimAction)
check(aaRes == XR_SUCCESS) { "xrCreateAction(aim) failed: $aaRes" }
val aimAction = XrAction(pAimAction[0], actionSet)
xrAimPoseAction = aimAction
// Select (boolean)
val selectInfo = XrActionCreateInfo.calloc(stack)
.type(XR_TYPE_ACTION_CREATE_INFO)
.actionType(XR_ACTION_TYPE_BOOLEAN_INPUT)
.actionName(stack.UTF8("select_click"))
.localizedActionName(stack.UTF8("Select"))
.countSubactionPaths(subactionPaths.remaining())
.subactionPaths(subactionPaths)
val pSelectAction = stack.callocPointer(1)
val saRes = xrCreateAction(actionSet, selectInfo, pSelectAction)
check(saRes == XR_SUCCESS) { "xrCreateAction(select) failed: $saRes" }
val selectAction = XrAction(pSelectAction[0], actionSet)
xrSelectAction = selectAction
fun createAction(
type: Int,
name: String,
localized: String,
): XrAction {
val subPaths = subactionPaths.duplicate()
subPaths.position(0)
val info = XrActionCreateInfo.calloc(stack)
.type(XR_TYPE_ACTION_CREATE_INFO)
.actionType(type)
.actionName(stack.UTF8(name))
.localizedActionName(stack.UTF8(localized))
.countSubactionPaths(subPaths.remaining())
.subactionPaths(subPaths)
val p = stack.callocPointer(1)
val r = xrCreateAction(actionSet, info, p)
check(r == XR_SUCCESS) { "xrCreateAction($name) failed: $r" }
return XrAction(p[0], actionSet)
}
// Full input set (Valve Index + HTC Vive)
val triggerValueAction = createAction(XR_ACTION_TYPE_FLOAT_INPUT, "trigger_value", "Trigger Value")
val triggerClickAction = createAction(XR_ACTION_TYPE_BOOLEAN_INPUT, "trigger_click", "Trigger Click")
val squeezeValueAction = createAction(XR_ACTION_TYPE_FLOAT_INPUT, "squeeze_value", "Squeeze Value")
val squeezeClickAction = createAction(XR_ACTION_TYPE_BOOLEAN_INPUT, "squeeze_click", "Squeeze Click")
val trackpadAction = createAction(XR_ACTION_TYPE_VECTOR2F_INPUT, "trackpad", "Trackpad")
val trackpadTouchAction = createAction(XR_ACTION_TYPE_BOOLEAN_INPUT, "trackpad_touch", "Trackpad Touch")
val trackpadClickAction = createAction(XR_ACTION_TYPE_BOOLEAN_INPUT, "trackpad_click", "Trackpad Click")
val thumbstickAction = createAction(XR_ACTION_TYPE_VECTOR2F_INPUT, "thumbstick", "Thumbstick")
val thumbstickTouchAction = createAction(XR_ACTION_TYPE_BOOLEAN_INPUT, "thumbstick_touch", "Thumbstick Touch")
val thumbstickClickAction = createAction(XR_ACTION_TYPE_BOOLEAN_INPUT, "thumbstick_click", "Thumbstick Click")
val aAction = createAction(XR_ACTION_TYPE_BOOLEAN_INPUT, "a_click", "A")
val bAction = createAction(XR_ACTION_TYPE_BOOLEAN_INPUT, "b_click", "B")
val menuAction = createAction(XR_ACTION_TYPE_BOOLEAN_INPUT, "menu_click", "Menu")
val hapticAction = createAction(XR_ACTION_TYPE_VIBRATION_OUTPUT, "haptic", "Haptic")
xrTriggerValueAction = triggerValueAction
xrTriggerClickAction = triggerClickAction
xrSqueezeValueAction = squeezeValueAction
xrSqueezeClickAction = squeezeClickAction
xrTrackpadAction = trackpadAction
xrTrackpadTouchAction = trackpadTouchAction
xrTrackpadClickAction = trackpadClickAction
xrThumbstickAction = thumbstickAction
xrThumbstickTouchAction = thumbstickTouchAction
xrThumbstickClickAction = thumbstickClickAction
xrAAction = aAction
xrBAction = bAction
xrMenuAction = menuAction
xrHapticAction = hapticAction
// Suggested bindings.
// NOTE: runtimes may reject the entire suggestion if any binding is invalid for the profile.
// We therefore:
// - suggest Index and Vive separately
// - retry with reduced sets if needed
val leftGripPosePath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/grip/pose")
val rightGripPosePath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/grip/pose")
val leftAimPosePath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/aim/pose")
val rightAimPosePath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/aim/pose")
val leftHapticPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/output/haptic")
val rightHapticPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/output/haptic")
fun suggestBindings(profileName: String, profile: Long, pairs: List<Pair<XrAction, Long>>): Int {
val bindings = XrActionSuggestedBinding.calloc(pairs.size, stack)
for (i in pairs.indices) {
bindings[i].action(pairs[i].first).binding(pairs[i].second)
}
val suggested = XrInteractionProfileSuggestedBinding.calloc(stack)
.type(XR_TYPE_INTERACTION_PROFILE_SUGGESTED_BINDING)
.interactionProfile(profile)
.suggestedBindings(bindings)
val r = xrSuggestInteractionProfileBindings(xrInstanceLocal, suggested)
if (Options.openXRDebugInput && r != XR_SUCCESS) {
System.err.println("OpenXR: xrSuggestInteractionProfileBindings failed for $profileName : $r")
}
return r
}
fun suggestWithFallback(profileName: String, profile: Long, attempts: List<List<Pair<XrAction, Long>>>) {
for (i in attempts.indices) {
val r = suggestBindings(profileName, profile, attempts[i])
if (r == XR_SUCCESS) {
if (Options.openXRDebugInput && i > 0) {
System.err.println("OpenXR: $profileName bindings succeeded after fallback #$i")
}
return
}
}
}
// --- Valve Index controller profile ---
// Common paths for Index:
// - trigger: /input/trigger/value and /input/trigger/click
// - squeeze: /input/squeeze/value (analog)
// - thumbstick: /input/thumbstick (vector2) + /click + /touch
// - trackpad: /input/trackpad (vector2) + /click + /touch
// - buttons: /input/a/click, /input/b/click
val leftTriggerValuePath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/trigger/value")
val rightTriggerValuePath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/trigger/value")
val leftTriggerClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/trigger/click")
val rightTriggerClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/trigger/click")
val leftSqueezeValuePath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/squeeze/value")
val rightSqueezeValuePath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/squeeze/value")
val leftTrackpadPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/trackpad")
val rightTrackpadPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/trackpad")
val leftTrackpadClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/trackpad/click")
val rightTrackpadClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/trackpad/click")
val leftTrackpadTouchPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/trackpad/touch")
val rightTrackpadTouchPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/trackpad/touch")
val leftThumbstickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/thumbstick")
val rightThumbstickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/thumbstick")
val leftThumbstickClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/thumbstick/click")
val rightThumbstickClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/thumbstick/click")
val leftThumbstickTouchPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/thumbstick/touch")
val rightThumbstickTouchPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/thumbstick/touch")
val rightAPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/a/click")
val rightBPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/b/click")
// Touch exposes an analog trigger, not /input/trigger/click. Suggest its native
// profile rather than relying on runtime remapping from Index or Vive bindings.
val touchBindings = listOf(
poseAction to leftGripPosePath, poseAction to rightGripPosePath,
aimAction to leftAimPosePath, aimAction to rightAimPosePath,
triggerValueAction to leftTriggerValuePath, triggerValueAction to rightTriggerValuePath,
squeezeValueAction to leftSqueezeValuePath, squeezeValueAction to rightSqueezeValuePath,
thumbstickAction to leftThumbstickPath, thumbstickAction to rightThumbstickPath,
thumbstickClickAction to leftThumbstickClickPath, thumbstickClickAction to rightThumbstickClickPath,
thumbstickTouchAction to leftThumbstickTouchPath, thumbstickTouchAction to rightThumbstickTouchPath,
aAction to stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/x/click"),
aAction to rightAPath,
bAction to stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/y/click"),
bAction to rightBPath,
menuAction to stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/menu/click"),
hapticAction to leftHapticPath, hapticAction to rightHapticPath,
)
suggestWithFallback("oculus/touch_controller", profileOculusTouch, listOf(touchBindings))
val indexFull = listOf(
poseAction to leftGripPosePath,
poseAction to rightGripPosePath,
aimAction to leftAimPosePath,
aimAction to rightAimPosePath,
selectAction to leftTriggerClickPath,
selectAction to rightTriggerClickPath,
triggerValueAction to leftTriggerValuePath,
triggerValueAction to rightTriggerValuePath,
triggerClickAction to leftTriggerClickPath,
triggerClickAction to rightTriggerClickPath,
squeezeValueAction to leftSqueezeValuePath,
squeezeValueAction to rightSqueezeValuePath,
trackpadAction to leftTrackpadPath,
trackpadAction to rightTrackpadPath,
trackpadClickAction to leftTrackpadClickPath,
trackpadClickAction to rightTrackpadClickPath,
trackpadTouchAction to leftTrackpadTouchPath,
trackpadTouchAction to rightTrackpadTouchPath,
thumbstickAction to leftThumbstickPath,
thumbstickAction to rightThumbstickPath,
thumbstickClickAction to leftThumbstickClickPath,
thumbstickClickAction to rightThumbstickClickPath,
thumbstickTouchAction to leftThumbstickTouchPath,
thumbstickTouchAction to rightThumbstickTouchPath,
aAction to rightAPath,
bAction to rightBPath,
hapticAction to leftHapticPath,
hapticAction to rightHapticPath,
)
val indexReduced = listOf(
poseAction to leftGripPosePath,
poseAction to rightGripPosePath,
aimAction to leftAimPosePath,
aimAction to rightAimPosePath,
triggerValueAction to leftTriggerValuePath,
triggerValueAction to rightTriggerValuePath,
triggerClickAction to leftTriggerClickPath,
triggerClickAction to rightTriggerClickPath,
squeezeValueAction to leftSqueezeValuePath,
squeezeValueAction to rightSqueezeValuePath,
thumbstickAction to leftThumbstickPath,
thumbstickAction to rightThumbstickPath,
thumbstickClickAction to leftThumbstickClickPath,
thumbstickClickAction to rightThumbstickClickPath,
hapticAction to leftHapticPath,
hapticAction to rightHapticPath,
)
val indexMinimum = listOf(
poseAction to leftGripPosePath,
poseAction to rightGripPosePath,
aimAction to leftAimPosePath,
aimAction to rightAimPosePath,
triggerClickAction to leftTriggerClickPath,
triggerClickAction to rightTriggerClickPath,
hapticAction to leftHapticPath,
hapticAction to rightHapticPath,
)
suggestWithFallback(
"valve/index_controller",
profileValveIndex,
listOf(indexFull, indexReduced, indexMinimum)
)
// --- HTC Vive controller profile ---
// Common paths for Vive:
// - trigger: /input/trigger/value and /input/trigger/click
// - grip button: /input/squeeze/click (boolean)
// - trackpad: /input/trackpad (vector2) + /click + /touch
// - menu: /input/menu/click
val leftViveTriggerValuePath = leftTriggerValuePath
val rightViveTriggerValuePath = rightTriggerValuePath
val leftViveTriggerClickPath = leftTriggerClickPath
val rightViveTriggerClickPath = rightTriggerClickPath
val leftViveSqueezeClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/squeeze/click")
val rightViveSqueezeClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/squeeze/click")
val leftViveMenuClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/left/input/menu/click")
val rightViveMenuClickPath = stringToPath(xrInstanceLocal, stack, "/user/hand/right/input/menu/click")
val viveFull = listOf(
poseAction to leftGripPosePath,
poseAction to rightGripPosePath,
aimAction to leftAimPosePath,
aimAction to rightAimPosePath,
selectAction to leftViveTriggerClickPath,
selectAction to rightViveTriggerClickPath,
triggerValueAction to leftViveTriggerValuePath,
triggerValueAction to rightViveTriggerValuePath,
triggerClickAction to leftViveTriggerClickPath,
triggerClickAction to rightViveTriggerClickPath,
squeezeClickAction to leftViveSqueezeClickPath,
squeezeClickAction to rightViveSqueezeClickPath,
trackpadAction to leftTrackpadPath,
trackpadAction to rightTrackpadPath,
trackpadClickAction to leftTrackpadClickPath,
trackpadClickAction to rightTrackpadClickPath,
trackpadTouchAction to leftTrackpadTouchPath,
trackpadTouchAction to rightTrackpadTouchPath,
menuAction to leftViveMenuClickPath,
menuAction to rightViveMenuClickPath,
hapticAction to leftHapticPath,
hapticAction to rightHapticPath,
)
val viveReduced = listOf(
poseAction to leftGripPosePath,
poseAction to rightGripPosePath,
aimAction to leftAimPosePath,
aimAction to rightAimPosePath,
triggerValueAction to leftViveTriggerValuePath,
triggerValueAction to rightViveTriggerValuePath,
triggerClickAction to leftViveTriggerClickPath,
triggerClickAction to rightViveTriggerClickPath,
trackpadAction to leftTrackpadPath,
trackpadAction to rightTrackpadPath,
trackpadClickAction to leftTrackpadClickPath,
trackpadClickAction to rightTrackpadClickPath,
hapticAction to leftHapticPath,
hapticAction to rightHapticPath,
)
val viveMinimum = listOf(
poseAction to leftGripPosePath,
poseAction to rightGripPosePath,
aimAction to leftAimPosePath,
aimAction to rightAimPosePath,
triggerClickAction to leftViveTriggerClickPath,
triggerClickAction to rightViveTriggerClickPath,
hapticAction to leftHapticPath,
hapticAction to rightHapticPath,
)
suggestWithFallback(
"htc/vive_controller",
profileHTCVive,
listOf(viveFull, viveReduced, viveMinimum)
)
// Eye gaze belongs to the same action set, but has no left/right hand subactions.
eyeGazeInteraction = ExtEyeGazeInteraction.create(session, xrSystemId, actionSet)
// Attach action set to the session
val actionSetsBuf = stack.callocPointer(1)
actionSetsBuf.put(0, actionSet.address())
val attachInfo = XrSessionActionSetsAttachInfo.calloc(stack)
.type(XR_TYPE_SESSION_ACTION_SETS_ATTACH_INFO)
.actionSets(actionSetsBuf)
val atRes = xrAttachSessionActionSets(session, attachInfo)
check(atRes == XR_SUCCESS) { "xrAttachSessionActionSets failed: $atRes" }
// Create action spaces for grip pose (left/right)
val identityPose = XrPosef.calloc(stack)
identityPose.orientation().w(1f)
val leftSpaceInfo = XrActionSpaceCreateInfo.calloc(stack)
.type(XR_TYPE_ACTION_SPACE_CREATE_INFO)
.action(poseAction)
.subactionPath(leftHandPath)
leftSpaceInfo.poseInActionSpace(identityPose)
val rightSpaceInfo = XrActionSpaceCreateInfo.calloc(stack)
.type(XR_TYPE_ACTION_SPACE_CREATE_INFO)
.action(poseAction)
.subactionPath(rightHandPath)
rightSpaceInfo.poseInActionSpace(identityPose)
val pLeftSpace = stack.callocPointer(1)
val lsRes = xrCreateActionSpace(session, leftSpaceInfo, pLeftSpace)
check(lsRes == XR_SUCCESS) { "xrCreateActionSpace(left) failed: $lsRes" }
leftGripSpace = XrSpace(pLeftSpace[0], session)
val pRightSpace = stack.callocPointer(1)
val rsRes = xrCreateActionSpace(session, rightSpaceInfo, pRightSpace)
check(rsRes == XR_SUCCESS) { "xrCreateActionSpace(right) failed: $rsRes" }
rightGripSpace = XrSpace(pRightSpace[0], session)
// Create action spaces for aim pose (left/right)
val leftAimInfo = XrActionSpaceCreateInfo.calloc(stack)
.type(XR_TYPE_ACTION_SPACE_CREATE_INFO)
.action(aimAction)
.subactionPath(leftHandPath)
leftAimInfo.poseInActionSpace(identityPose)
val rightAimInfo = XrActionSpaceCreateInfo.calloc(stack)
.type(XR_TYPE_ACTION_SPACE_CREATE_INFO)
.action(aimAction)
.subactionPath(rightHandPath)
rightAimInfo.poseInActionSpace(identityPose)
val pLeftAim = stack.callocPointer(1)
val laRes = xrCreateActionSpace(session, leftAimInfo, pLeftAim)
check(laRes == XR_SUCCESS) { "xrCreateActionSpace(leftAim) failed: $laRes" }
leftAimSpace = XrSpace(pLeftAim[0], session)
val pRightAim = stack.callocPointer(1)
val raRes = xrCreateActionSpace(session, rightAimInfo, pRightAim)
check(raRes == XR_SUCCESS) { "xrCreateActionSpace(rightAim) failed: $raRes" }
rightAimSpace = XrSpace(pRightAim[0], session)
}
}
private fun stringToPath(xrInstanceLocal: XrInstance, stack: MemoryStack, str: String): Long {
val out = stack.callocLong(1)
val res = xrStringToPath(xrInstanceLocal, str, out)
check(res == XR_SUCCESS) { "xrStringToPath('$str') failed: $res" }
return out[0]
}
private fun initHandTracking(session: XrSession) {
if (session.getCapabilities().xrCreateHandTrackerEXT == 0L) return
MemoryStack.stackPush().use { stack ->
val createInfo = XrHandTrackerCreateInfoEXT.calloc(stack)
.type(EXTHandTracking.XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT)
.hand(EXTHandTracking.XR_HAND_LEFT_EXT)
.handJointSet(EXTHandTracking.XR_HAND_JOINT_SET_DEFAULT_EXT)
val pTracker = stack.callocPointer(1)
var res = EXTHandTracking.xrCreateHandTrackerEXT(session, createInfo, pTracker)
if (res == XR_SUCCESS) leftHandTracker = XrHandTrackerEXT(pTracker[0], session)
else lprintln("warning,openxr", "xrCreateHandTrackerEXT (left) failed: $res")
createInfo.hand(EXTHandTracking.XR_HAND_RIGHT_EXT)
res = EXTHandTracking.xrCreateHandTrackerEXT(session, createInfo, pTracker)
if (res == XR_SUCCESS) rightHandTracker = XrHandTrackerEXT(pTracker[0], session)
else lprintln("warning,openxr", "xrCreateHandTrackerEXT (right) failed: $res")
}
handTrackingEnabled = leftHandTracker != null || rightHandTracker != null
if (handTrackingEnabled) lprintln("info,openxr", "hand tracking enabled")
}
private fun syncHandTracking(session: XrSession, space: XrSpace, predictedTime: Long) {
MemoryStack.stackPush().use { stack ->
val locateInfo = XrHandJointsLocateInfoEXT.calloc(stack)
.type(EXTHandTracking.XR_TYPE_HAND_JOINTS_LOCATE_INFO_EXT)
.baseSpace(space)
.time(predictedTime)
leftJointsBuf?.let { syncHandJoints(stack, leftHandTracker, locateInfo, it, leftHandState) }
rightJointsBuf?.let { syncHandJoints(stack, rightHandTracker, locateInfo, it, rightHandState) }
}
}
private fun syncHandJoints(
stack: MemoryStack,
tracker: XrHandTrackerEXT?,
locateInfo: XrHandJointsLocateInfoEXT,
buf: XrHandJointLocationEXT.Buffer,
state: HandTrackingState,
) {
if (tracker == null) { state.isActive = false; return }
val locations = XrHandJointLocationsEXT.calloc(stack)
.type(EXTHandTracking.XR_TYPE_HAND_JOINT_LOCATIONS_EXT)
.jointLocations(buf)
val res = EXTHandTracking.xrLocateHandJointsEXT(tracker, locateInfo, locations)
if (res != XR_SUCCESS || !locations.isActive()) {
state.isActive = false
state.pinchStrength = 0f
return
}
state.isActive = true
for (i in 0 until HandJoint.COUNT) {
val src = buf[i]
val flags = src.locationFlags()
val dst = state.joints[i]
dst.positionValid = (flags and XR_SPACE_LOCATION_POSITION_VALID_BIT.toLong()) != 0L
dst.orientationValid = (flags and XR_SPACE_LOCATION_ORIENTATION_VALID_BIT.toLong()) != 0L
dst.radius = src.radius()
if (dst.positionValid) {
val p = src.pose().`position$`()
dst.position.x = p.x()
dst.position.y = p.y()
dst.position.z = p.z()
}
if (dst.orientationValid) {
val o = src.pose().orientation()
dst.orientation.x = o.x()
dst.orientation.y = o.y()
dst.orientation.z = o.z()
dst.orientation.w = o.w()
}
}
// Pinch strength: 0.0=open (50mm gap), 1.0=pinched (10mm gap).
val thumbTip = state[HandJoint.THUMB_TIP]
val indexTip = state[HandJoint.INDEX_TIP]
state.pinchStrength = if (thumbTip.positionValid && indexTip.positionValid) {
val dx = thumbTip.position.x - indexTip.position.x
val dy = thumbTip.position.y - indexTip.position.y
val dz = thumbTip.position.z - indexTip.position.z
val dist = kotlin.math.sqrt((dx * dx + dy * dy + dz * dz).toDouble()).toFloat()
((0.05f - dist) / 0.04f).coerceIn(0f, 1f)
} else 0f
}
Body tracking (WiVRn generic trackers over XR_MNDX_xdev_space)
Sets up body tracking, if the runtime offers any.
The obvious route, XR_FB_body_tracking, is a dead end under WiVRn: its server only wires
Monado's static_roles.body when a SolarXR/SlimeVR device is present
(server/driver/wivrn_session.cpp, inside #if WIVRN_FEATURE_SOLARXR), so
xrCreateBodyTrackerFB reports unsupported for a bare Quest 3 even though the runtime
advertises the extension. What the headset's own inside-out body tracking actually gives us
is a handful of anonymous generic trackers, and those are only reachable through the Monado
vendor extension XR_MNDX_xdev_space.
private fun initBodyTracking(session: XrSession) {
if (MndxXDevSpace.EXTENSION_NAME !in Instance.enabledOpenXRInstanceExtensions) return
val instance = xrInstance ?: return
val ext = MndxXDevSpace.load(instance) ?: return
val list = ext.createXDevList(session) ?: return
xdevSpaceExt = ext
xdevList = list
refreshBodyTrackers(session)
// Keep the extension loaded even with no trackers yet: the user can turn body tracking on
// in the headset mid-session, and refreshBodyTrackers picks that up from the generation
// number. bodyTrackingEnabled tracks whether we currently have any.
}
Re-enumerates the xdev list and rebuilds our tracker spaces. Cheap enough to gate on the generation number, which the runtime bumps whenever devices come or go.
private fun refreshBodyTrackers(session: XrSession) {
val ext = xdevSpaceExt ?: return
if (xdevList == 0L) return
val generation = ext.generationNumber(xdevList) ?: return
if (generation == xdevGeneration) return
xdevGeneration = generation
bodyTrackers.forEach { xrDestroySpace(it.space) }
bodyTrackers = emptyList()
val found = mutableListOf<BodyTracker>()
for (id in ext.enumerateXDevs(xdevList)) {
val props = ext.getProperties(xdevList, id) ?: continue
if (!props.canCreateSpace) continue
// WiVRn names its body trackers "WiVRn Generic Tracker #n" with serial "wivrn-n", and
// that n is the whitelist index we need. Matching on the serial also filters out the
// HMD, the controllers, and any SolarXR trackers sharing the list.
val index = WIVRN_TRACKER_SERIAL.matchEntire(props.serial)?.groupValues?.get(1)?.toIntOrNull() ?: continue
val space = ext.createXDevSpace(session, xdevList, id) ?: continue
found += BodyTracker(index - 1, space) // serials are 1-based
}
found.sortBy { it.index }
bodyTrackers = found
bodyJointMapping = inferBodyJointMapping(found.size)
bodyTrackingState.trackedJoints = bodyJointMapping.toSet()
bodyTrackingEnabled = found.isNotEmpty()
if (found.isEmpty()) {
bodyTrackingState.isActive = false
bodyTrackingState.joints.forEach { it.positionValid = false; it.orientationValid = false }
} else {
lprintln(
"info,openxr",
"body tracking enabled: ${found.size} tracker(s) -> " +
if (bodyJointMapping.isEmpty()) "unmapped (use bodyTrackerPose)" else "$bodyJointMapping"
)
}
}
Works out which joint each tracker is, from how many there are.
WiVRn's client forwards a fixed whitelist filtered by two user toggles, preserving order
(client/xr/fb_body_tracker.cpp: get_whitelisted_joints). Only three counts are reachable,
and each pins the mapping exactly:
- 3: lower body off — chest and both lower arms.
- 7: lower body on, hip off — the above plus lower legs and feet.
- 8: lower body and hip on — everything.
Any other count means we're not talking to a WiVRn fb-body-tracking client (an HTC or Pico tracker set, or SolarXR devices), so we decline to guess and callers fall back to bodyTrackerPose.
source — 1101 lines, from line 1810
private fun inferBodyJointMapping(trackerCount: Int): List<BodyJoint> {
val all = BodyJoint.entries
val withoutHips = all.filter { it != BodyJoint.HIPS }
val upperBodyOnly = withoutHips.filter {
it == BodyJoint.CHEST || it == BodyJoint.LEFT_ARM_LOWER || it == BodyJoint.RIGHT_ARM_LOWER
}
return when (trackerCount) {
upperBodyOnly.size -> upperBodyOnly // 3
withoutHips.size -> withoutHips // 7
all.size -> all // 8
else -> emptyList()
}
}
private fun syncBodyTracking(session: XrSession, space: XrSpace, predictedTime: Long) {
refreshBodyTrackers(session)
if (bodyTrackers.isEmpty()) return
MemoryStack.stackPush().use { stack ->
val location = XrSpaceLocation.calloc(stack).type(XR_TYPE_SPACE_LOCATION)
var anyValid = false
for (tracker in bodyTrackers) {
val dst = tracker.pose
val res = xrLocateSpace(tracker.space, space, predictedTime, location)
if (res != XR_SUCCESS) {
dst.positionValid = false
dst.orientationValid = false
continue
}
val flags = location.locationFlags()
dst.positionValid = (flags and XR_SPACE_LOCATION_POSITION_VALID_BIT.toLong()) != 0L
dst.orientationValid = (flags and XR_SPACE_LOCATION_ORIENTATION_VALID_BIT.toLong()) != 0L
if (dst.positionValid) {
val p = location.pose().`position$`()
dst.position.x = p.x()
dst.position.y = p.y()
dst.position.z = p.z()
}
if (dst.orientationValid) {
val o = location.pose().orientation()
dst.orientation.x = o.x()
dst.orientation.y = o.y()
dst.orientation.z = o.z()
dst.orientation.w = o.w()
}
anyValid = anyValid || dst.positionValid || dst.orientationValid
// Mirror into the semantic slot, when we know which one that is.
val joint = bodyJointMapping.getOrNull(tracker.index) ?: continue
val slot = bodyTrackingState[joint]
slot.positionValid = dst.positionValid
slot.orientationValid = dst.orientationValid
if (dst.positionValid) slot.position.set(dst.position)
if (dst.orientationValid) slot.orientation.set(dst.orientation)
}
bodyTrackingState.isActive = anyValid
}
}
private fun syncAndUpdateControllers(session: XrSession, referenceSpace: XrSpace, predictedDisplayTime: Long) {
val actionSet = xrActionSet ?: return
val gripAction = xrPoseAction ?: return
val aimAction = xrAimPoseAction
MemoryStack.stackPush().use { stack ->
val activeSets = XrActiveActionSet.calloc(1, stack)
activeSets[0].actionSet(actionSet).subactionPath(XR_NULL_PATH)
val syncInfo = XrActionsSyncInfo.calloc(stack)
.type(XR_TYPE_ACTIONS_SYNC_INFO)
.activeActionSets(activeSets)
val syncResult = xrSyncActions(session, syncInfo)
if (eyeGazeInteraction != null && syncResult == XR_SUCCESS && sessionState == XR_SESSION_STATE_FOCUSED) {
CPUSpan.time("xr_eye_gaze") {
eyeGazeInteraction?.update(session, referenceSpace, predictedDisplayTime)
}
}
updateControllerForHand(
session,
referenceSpace,
predictedDisplayTime,
gripAction,
aimAction,
leftHandPath,
leftGripSpace,
leftAimSpace,
leftController
)
updateControllerForHand(
session,
referenceSpace,
predictedDisplayTime,
gripAction,
aimAction,
rightHandPath,
rightGripSpace,
rightAimSpace,
rightController
)
if (Options.openXRDebugInput && !inputDebugPrinted) {
if (!leftController.active && !rightController.active) {
printInputDiagnostics(session)
inputDebugPrinted = true
}
}
}
}
private fun printInputDiagnostics(session: XrSession) {
val xrInstanceLocal = xrInstance ?: return
val actionSet = xrActionSet ?: return
val gripAction = xrPoseAction
val aimAction = xrAimPoseAction
MemoryStack.stackPush().use { stack ->
fun pathToString(path: Long): String {
if (path == XR_NULL_PATH) return "<null>"
val len = stack.callocInt(1)
// First call to get required size
xrPathToString(xrInstanceLocal, path, len, null)
val buf = stack.malloc(len[0])
xrPathToString(xrInstanceLocal, path, len, buf)
return MemoryUtil.memUTF8(buf, len[0] - 1)
}
fun printProfile(userPathString: String) {
val userPath = stringToPath(xrInstanceLocal, stack, userPathString)
val info = XrInteractionProfileState.calloc(stack).type(XR_TYPE_INTERACTION_PROFILE_STATE)
val r = xrGetCurrentInteractionProfile(session, userPath, info)
if (r == XR_SUCCESS) {
System.err.println("OpenXR input: $userPathString profile=${pathToString(info.interactionProfile())}")
} else {
System.err.println("OpenXR input: xrGetCurrentInteractionProfile failed for $userPathString : $r")
}
}
fun printBoundSources(action: XrAction?, label: String) {
if (action == null) {
System.err.println("OpenXR input: action $label is null")
return
}
val getInfo = XrBoundSourcesForActionEnumerateInfo.calloc(stack)
.type(XR_TYPE_BOUND_SOURCES_FOR_ACTION_ENUMERATE_INFO)
.action(action)
val count = stack.callocInt(1)
val r0 = xrEnumerateBoundSourcesForAction(session, getInfo, count, null)
if (r0 != XR_SUCCESS) {
System.err.println("OpenXR input: xrEnumerateBoundSourcesForAction(count) failed for $label : $r0")
return
}
val paths = stack.callocLong(count[0])
val r1 = xrEnumerateBoundSourcesForAction(session, getInfo, count, paths)
if (r1 != XR_SUCCESS) {
System.err.println("OpenXR input: xrEnumerateBoundSourcesForAction(paths) failed for $label : $r1")
return
}
val sources = (0 until count[0]).joinToString { i -> pathToString(paths[i]) }
System.err.println("OpenXR input: bound sources for $label: [$sources]")
}
System.err.println("OpenXR input: actionSet=${actionSet.address()}")
printProfile("/user/hand/left")
printProfile("/user/hand/right")
printBoundSources(gripAction, "grip_pose")
printBoundSources(aimAction, "aim_pose")
printBoundSources(xrSelectAction, "select")
printBoundSources(xrTriggerValueAction, "trigger_value")
printBoundSources(xrTriggerClickAction, "trigger_click")
printBoundSources(xrSqueezeValueAction, "squeeze_value")
printBoundSources(xrSqueezeClickAction, "squeeze_click")
printBoundSources(xrTrackpadAction, "trackpad")
printBoundSources(xrTrackpadTouchAction, "trackpad_touch")
printBoundSources(xrTrackpadClickAction, "trackpad_click")
printBoundSources(xrThumbstickAction, "thumbstick")
printBoundSources(xrThumbstickTouchAction, "thumbstick_touch")
printBoundSources(xrThumbstickClickAction, "thumbstick_click")
printBoundSources(xrAAction, "a")
printBoundSources(xrBAction, "b")
printBoundSources(xrMenuAction, "menu")
printBoundSources(xrHapticAction, "haptic")
}
}
private fun updateControllerForHand(
session: XrSession,
referenceSpace: XrSpace,
predictedDisplayTime: Long,
gripPoseAction: XrAction,
aimPoseAction: XrAction?,
subactionPath: Long,
gripSpace: XrSpace?,
aimSpace: XrSpace?,
out: ControllerState
) {
MemoryStack.stackPush().use { stack ->
// Grip pose active?
val gripGetInfo = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(gripPoseAction)
.subactionPath(subactionPath)
val gripState = XrActionStatePose.calloc(stack).type(XR_TYPE_ACTION_STATE_POSE)
xrGetActionStatePose(session, gripGetInfo, gripState)
// Aim pose active?
var aimActive = false
if (aimPoseAction != null) {
val aimGetInfo = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(aimPoseAction)
.subactionPath(subactionPath)
val aimState = XrActionStatePose.calloc(stack).type(XR_TYPE_ACTION_STATE_POSE)
xrGetActionStatePose(session, aimGetInfo, aimState)
aimActive = aimState.isActive()
}
val gripActive = gripState.isActive()
var anyActive = gripActive || aimActive
// Generic select (usually trigger/click)
run {
val action = xrSelectAction
if (action != null) {
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
anyActive = anyActive || st.isActive()
out.selectPressed = st.currentState()
} else {
out.selectPressed = false
}
}
// Trigger
run {
var valueActive = false
var value = 0f
xrTriggerValueAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateFloat.calloc(stack).type(XR_TYPE_ACTION_STATE_FLOAT)
xrGetActionStateFloat(session, gi, st)
valueActive = st.isActive()
value = st.currentState()
}
var clickActive = false
var pressed = false
xrTriggerClickAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
clickActive = st.isActive()
pressed = st.currentState()
}
anyActive = anyActive || valueActive || clickActive
out.triggerValue = if (valueActive) value else 0f
out.triggerPressed = pressed || (valueActive && value >= if (out.triggerPressed) 0.45f else 0.55f)
}
// Squeeze / grip
run {
var valueActive = false
var value = 0f
xrSqueezeValueAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateFloat.calloc(stack).type(XR_TYPE_ACTION_STATE_FLOAT)
xrGetActionStateFloat(session, gi, st)
valueActive = st.isActive()
value = st.currentState()
}
var clickActive = false
var pressed = false
xrSqueezeClickAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
clickActive = st.isActive()
pressed = st.currentState()
}
anyActive = anyActive || valueActive || clickActive
out.squeezeValue = if (valueActive) value else 0f
out.squeezePressed = pressed || (valueActive && value > 0.5f)
}
// Trackpad
run {
var vecActive = false
var x = 0f
var y = 0f
xrTrackpadAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateVector2f.calloc(stack).type(XR_TYPE_ACTION_STATE_VECTOR2F)
xrGetActionStateVector2f(session, gi, st)
vecActive = st.isActive()
val v = st.currentState()
x = v.x(); y = v.y()
}
var touchActive = false
var touched = false
xrTrackpadTouchAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
touchActive = st.isActive()
touched = st.currentState()
}
var clickActive = false
var pressed = false
xrTrackpadClickAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
clickActive = st.isActive()
pressed = st.currentState()
}
anyActive = anyActive || vecActive || touchActive || clickActive
out.trackpadX = if (vecActive) x else 0f
out.trackpadY = if (vecActive) y else 0f
out.trackpadTouched = touched
out.trackpadPressed = pressed
}
// Thumbstick
run {
var vecActive = false
var x = 0f
var y = 0f
xrThumbstickAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateVector2f.calloc(stack).type(XR_TYPE_ACTION_STATE_VECTOR2F)
xrGetActionStateVector2f(session, gi, st)
vecActive = st.isActive()
val v = st.currentState()
x = v.x(); y = v.y()
}
var touchActive = false
var touched = false
xrThumbstickTouchAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
touchActive = st.isActive()
touched = st.currentState()
}
var clickActive = false
var pressed = false
xrThumbstickClickAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
clickActive = st.isActive()
pressed = st.currentState()
}
anyActive = anyActive || vecActive || touchActive || clickActive
out.thumbstickX = if (vecActive) x else 0f
out.thumbstickY = if (vecActive) y else 0f
out.thumbstickTouched = touched
out.thumbstickPressed = pressed
}
// Buttons
run {
var aActive = false
var aPressed = false
xrAAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
aActive = st.isActive()
aPressed = st.currentState()
}
var bActive = false
var bPressed = false
xrBAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
bActive = st.isActive()
bPressed = st.currentState()
}
var menuActive = false
var menuPressed = false
xrMenuAction?.let { action ->
val gi = XrActionStateGetInfo.calloc(stack)
.type(XR_TYPE_ACTION_STATE_GET_INFO)
.action(action)
.subactionPath(subactionPath)
val st = XrActionStateBoolean.calloc(stack).type(XR_TYPE_ACTION_STATE_BOOLEAN)
xrGetActionStateBoolean(session, gi, st)
menuActive = st.isActive()
menuPressed = st.currentState()
}
anyActive = anyActive || aActive || bActive || menuActive
out.aPressed = aPressed
out.bPressed = bPressed
out.menuPressed = menuPressed
}
out.active = anyActive
// Choose the best pose space to locate.
val chosenSpace = if (aimActive && aimSpace != null) aimSpace else gripSpace
if (!out.active || chosenSpace == null) {
out.poseValid = false
return@use
}
val loc = XrSpaceLocation.calloc(stack).type(XR_TYPE_SPACE_LOCATION)
val res = xrLocateSpace(chosenSpace, referenceSpace, predictedDisplayTime, loc)
if (res != XR_SUCCESS) {
out.poseValid = false
return@use
}
val flags = loc.locationFlags()
val posValid = (flags and XR_SPACE_LOCATION_POSITION_VALID_BIT.toLong()) != 0L
val oriValid = (flags and XR_SPACE_LOCATION_ORIENTATION_VALID_BIT.toLong()) != 0L
out.poseValid = posValid && oriValid
if (!out.poseValid) return@use
val p = loc.pose().`position$`()
val q = loc.pose().orientation()
out.position.x = p.x()
out.position.y = p.y()
out.position.z = p.z()
out.rotation.x = q.x()
out.rotation.y = q.y()
out.rotation.z = q.z()
out.rotation.w = q.w()
}
}
private fun createImageView(image: Long, format: Int): Long {
MemoryStack.stackPush().use { stack ->
val viewInfo = VkImageViewCreateInfo.calloc(stack)
.sType(VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO)
.image(image)
.viewType(VK_IMAGE_VIEW_TYPE_2D)
.format(format)
viewInfo.subresourceRange()
.aspectMask(VK_IMAGE_ASPECT_COLOR_BIT)
.baseMipLevel(0)
.levelCount(1)
.baseArrayLayer(0)
.layerCount(1)
val pView = stack.callocLong(1)
val res = vkCreateImageView(renderer.device.vkDevice, viewInfo, null, pView)
check(res == VK_SUCCESS) { "vkCreateImageView failed: $res" }
return pView[0]
}
}
private fun renderEyesVulkan(acquiredIndices: IntArray) {
// Ensure previous submission is complete before reusing the command buffer/fence.
// (If we didn't do a hard wait before release, this is where we naturally apply backpressure.)
if (!fence.isSignaled()) {
fence.fenceWait()
}
fence.reset()
cmd.reset()
cmd.beginRecording()
// Begin GPU profiling for this frame
val ps = profilerState
var eyesScope: GpuProfiler.PipeStatScope? = null
var perfScope: GpuProfiler.PerfScope? = null
ps?.let { p ->
synchronized(p.lock) {
p.markers.tsFrameBegin = p.profiler.writeTimestamp2(
cmd.vkCommandBuffer, 0, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT
)
eyesScope = p.profiler.beginPipelineStats(cmd.vkCommandBuffer, 0)
eyesScope?.let { p.markers.pipeStatsEyes = it.index }
perfScope = p.profiler.beginPerfCounters(cmd.vkCommandBuffer, 0)
}
}
val arraySource = layeredSource
val sourceIsLayered = arraySource != null
val eyeSource = arraySource ?: sideBySideSource
if (eyeSource == null) {
// Upload dirty mesh buffers once per frame BEFORE beginning dynamic rendering.
// This keeps VertexBuffer transfers outside the render pass instance.
modules.forEach { (_, mp) ->
mp.current.meshes.values.forEach { mesh ->
mesh.prepareForDraw(cmd)
}
}
}
MemoryStack.stackPush().use { stack ->
if (eyeSource != null) {
// Copy the two source views into runtime-owned eye swapchains.
check(swapchains.size >= 2) { "Need at least two swapchains for stereo" }
val left = swapchains[0]
val right = swapchains[1]
val eyeW = left.width
val eyeH = left.height
check(eyeW == right.width && eyeH == right.height) {
"Expected stereo eyes to share the same size; got L=${left.width}x${left.height}, R=${right.width}x${right.height}"
}
val srcView = eyeSource.imageView
val srcImage = srcView.image
val srcUnderlying = srcView.underlyingImage
srcUnderlying?.let { img ->
val w = img.imageData.width
val h = img.imageData.height
require(w >= eyeW * (if (sourceIsLayered) 1 else 2) && h >= eyeH) {
"Stereo source too small: got ${w}x${h}, need at least ${eyeW * (if (sourceIsLayered) 1 else 2)}x${eyeH}"
}
}
val srcRange = VkImageSubresourceRange.calloc(stack)
.aspectMask(VK_IMAGE_ASPECT_COLOR_BIT)
.baseMipLevel(0)
.levelCount(1)
.baseArrayLayer(0)
.layerCount(if (sourceIsLayered) 2 else 1)
// The source may be a sampled RTT (SHADER_READ_ONLY_OPTIMAL) or a storage/render
// image (commonly GENERAL). Preserve the producer's tracked layout rather than
// assuming every HasImageView came from the default RTT path.
val srcOldLayout = srcUnderlying?.getLastKnownLayout()
?: VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL
val (srcStage, srcAccess) = when (srcOldLayout) {
VK_IMAGE_LAYOUT_GENERAL ->
VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT to
(VK_ACCESS_2_MEMORY_READ_BIT or VK_ACCESS_2_MEMORY_WRITE_BIT)
VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL ->
VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT to
VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL ->
VK_PIPELINE_STAGE_2_TRANSFER_BIT to VK_ACCESS_2_TRANSFER_WRITE_BIT
VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL ->
VK_PIPELINE_STAGE_2_TRANSFER_BIT to VK_ACCESS_2_TRANSFER_READ_BIT
else ->
VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT to VK_ACCESS_2_SHADER_READ_BIT
}
if (srcUnderlying != null)
image_layout_transition(
cmd.vkCommandBuffer,
srcUnderlying,
srcStage,
VK_PIPELINE_STAGE_2_TRANSFER_BIT,
srcAccess,
VK_ACCESS_2_TRANSFER_READ_BIT,
srcOldLayout,
VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
srcRange
)
else
image_layout_transition(
cmd.vkCommandBuffer,
srcImage,
srcStage,
VK_PIPELINE_STAGE_2_TRANSFER_BIT,
srcAccess,
VK_ACCESS_2_TRANSFER_READ_BIT,
srcOldLayout,
VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
srcRange
)
for (eye in 0..1) {
val sc = swapchains[eye]
val imageIndex = acquiredIndices[eye]
if (imageIndex < 0) continue
val dstImage = sc.images[imageIndex]
val dstRange = VkImageSubresourceRange.calloc(stack)
.aspectMask(VK_IMAGE_ASPECT_COLOR_BIT)
.baseMipLevel(0)
.levelCount(1)
.baseArrayLayer(0)
.layerCount(1)
val dstOldLayout = if (sc.colorSeen[imageIndex]) VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL else VK_IMAGE_LAYOUT_UNDEFINED
val (dstSrcStage, dstSrcAccess) = if (sc.colorSeen[imageIndex]) {
VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT to VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT
} else {
VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT to 0L
}
image_layout_transition(
cmd.vkCommandBuffer,
dstImage,
dstSrcStage,
VK_PIPELINE_STAGE_2_TRANSFER_BIT,
dstSrcAccess,
VK_ACCESS_2_TRANSFER_WRITE_BIT,
dstOldLayout,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
dstRange
)
val sourceEye = StereoEyeCopy.source(eye, eyeW, sourceIsLayered)
val region = VkImageCopy.calloc(1, stack)
region[0].srcSubresource()
.aspectMask(VK_IMAGE_ASPECT_COLOR_BIT)
.mipLevel(0)
.baseArrayLayer(sourceEye.layer)
.layerCount(1)
region[0].dstSubresource()
.aspectMask(VK_IMAGE_ASPECT_COLOR_BIT)
.mipLevel(0)
.baseArrayLayer(0)
.layerCount(1)
region[0].srcOffset().set(sourceEye.x, 0, 0)
region[0].dstOffset().set(0, 0, 0)
region[0].extent().set(eyeW, eyeH, 1)
vkCmdCopyImage(
cmd.vkCommandBuffer,
srcImage,
vkLayout(VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL),
dstImage,
vkLayout(VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL),
region
)
image_layout_transition(
cmd.vkCommandBuffer,
dstImage,
VK_PIPELINE_STAGE_2_TRANSFER_BIT,
VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT,
VK_ACCESS_2_TRANSFER_WRITE_BIT,
VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT,
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
dstRange
)
sc.colorSeen[imageIndex] = true
}
val (restoreStage, restoreAccess) = when (srcOldLayout) {
VK_IMAGE_LAYOUT_GENERAL ->
VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT to
(VK_ACCESS_2_MEMORY_READ_BIT or VK_ACCESS_2_MEMORY_WRITE_BIT)
VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL ->
VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT to
VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT
VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL ->
VK_PIPELINE_STAGE_2_TRANSFER_BIT to VK_ACCESS_2_TRANSFER_WRITE_BIT
VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL ->
VK_PIPELINE_STAGE_2_TRANSFER_BIT to VK_ACCESS_2_TRANSFER_READ_BIT
else ->
VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT to VK_ACCESS_2_SHADER_READ_BIT
}
// Restore the producer's layout (e.g. GENERAL for a storage image, shader-read for
// a conventional RTT) so subsequent in-app monitoring uses the expected layout.
if (srcUnderlying != null) {
image_layout_transition(
cmd.vkCommandBuffer,
srcUnderlying,
VK_PIPELINE_STAGE_2_TRANSFER_BIT,
restoreStage,
VK_ACCESS_2_TRANSFER_READ_BIT,
restoreAccess,
VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
srcOldLayout,
srcRange
)
} else {
image_layout_transition(
cmd.vkCommandBuffer,
srcImage,
VK_PIPELINE_STAGE_2_TRANSFER_BIT,
restoreStage,
VK_ACCESS_2_TRANSFER_READ_BIT,
restoreAccess,
VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
srcOldLayout,
srcRange
)
}
} else {
for (eye in swapchains.indices) {
currentEyeIndex = eye
val sc = swapchains[eye]
val imageIndex = acquiredIndices[eye]
if (imageIndex < 0) continue
// Per-eye GPU timing markers
val eyeMarks = XrEyeMarks(eyeIndex = eye)
ps?.let { p ->
synchronized(p.lock) {
eyeMarks.tsBegin = p.profiler.writeTimestamp2(
cmd.vkCommandBuffer, 0, VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT
)
}
}
val width = sc.width
val height = sc.height
// Color transition (UNDEFINED -> COLOR_ATTACHMENT_OPTIMAL on first use; otherwise keep as COLOR_ATTACHMENT_OPTIMAL)
val colorRange = VkImageSubresourceRange.calloc(stack)
.aspectMask(VK_IMAGE_ASPECT_COLOR_BIT)
.baseMipLevel(0)
.levelCount(1)
.baseArrayLayer(0)
.layerCount(1)
val colorOldLayout =
if (sc.colorSeen[imageIndex]) VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL else VK_IMAGE_LAYOUT_UNDEFINED
val (srcStage, srcAccess) = if (sc.colorSeen[imageIndex]) {
VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT to VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT
} else {
VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT to 0L
}
image_layout_transition(
cmd.vkCommandBuffer,
sc.images[imageIndex],
srcStage,
VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT,
srcAccess,
VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT,
colorOldLayout,
VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
colorRange
)
sc.colorSeen[imageIndex] = true
// Depth transition
val depthAtt = sc.depthAttachments[imageIndex]
val depthRange = VkImageSubresourceRange.calloc(stack)
.aspectMask(Options.depthAspectMask(depthAtt.image.format))
.baseMipLevel(0)
.levelCount(1)
.baseArrayLayer(0)
.layerCount(1)
val depthOldLayout = if (sc.depthSeen[imageIndex]) VK_IMAGE_LAYOUT_DEPTH_ATTACHMENT_OPTIMAL else VK_IMAGE_LAYOUT_UNDEFINED
image_layout_transition(
cmd.vkCommandBuffer,
depthAtt.image,
VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT,
VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT,
0L,
VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT,
depthOldLayout,
VK_IMAGE_LAYOUT_DEPTH_ATTACHMENT_OPTIMAL,
depthRange
)
sc.depthSeen[imageIndex] = true
val colorAttachment = VkRenderingAttachmentInfoKHR.calloc(1, stack)
.sType(KHRDynamicRendering.VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO_KHR)
.imageView(sc.imageViews[imageIndex])
.imageLayout(vkLayout(VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL))
.resolveMode(VK_RESOLVE_MODE_NONE)
.loadOp(VK_ATTACHMENT_LOAD_OP_CLEAR)
.storeOp(VK_ATTACHMENT_STORE_OP_STORE)
.clearValue(
VkClearValue.calloc(stack).color(
VkClearColorValue.calloc(stack)
.float32(0, clearColor.getOrElse(0) { 1f })
.float32(1, clearColor.getOrElse(1) { 0f })
.float32(2, clearColor.getOrElse(2) { 1f })
.float32(3, clearColor.getOrElse(3) { 1f })
)
)
val depthAttachment = VkRenderingAttachmentInfoKHR.calloc(1, stack)
.sType(KHRDynamicRendering.VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO_KHR)
.imageView(depthAtt.imageView.vkImageView)
.imageLayout(vkLayout(VK_IMAGE_LAYOUT_DEPTH_ATTACHMENT_OPTIMAL))
.resolveMode(VK_RESOLVE_MODE_NONE)
.loadOp(VK_ATTACHMENT_LOAD_OP_CLEAR)
.storeOp(VK_ATTACHMENT_STORE_OP_DONT_CARE)
.clearValue(
VkClearValue.calloc(stack).depthStencil(
VkClearDepthStencilValue.calloc(stack)
.depth(1f)
.stencil(0)
)
)
val renderInfo = VkRenderingInfoKHR.calloc(stack)
.sType(KHRDynamicRendering.VK_STRUCTURE_TYPE_RENDERING_INFO_KHR)
.renderArea(
VkRect2D.calloc(stack)
.offset(VkOffset2D.calloc(stack).x(0).y(0))
.extent(VkExtent2D.calloc(stack).width(width).height(height))
)
.layerCount(1)
.pColorAttachments(colorAttachment)
.pDepthAttachment(depthAttachment[0])
modules.forEach { (_, mp) ->
mp.current.meshes.values.forEach { mesh ->
mesh.prepareForDraw(cmd)
mesh.push.primeBufferReferences(cmd) {
throw MissingRefException(afield.graphics.vulkan.buffers.unassignedMemberMessage(it))
}
}
mp.current.liveUniformStructures.values.forEach { structure ->
structure.primeBufferReferences(cmd) {
throw MissingRefException(afield.graphics.vulkan.buffers.unassignedMemberMessage(it))
}
}
}
meshModules.forEach { (_, mp) ->
mp.current.meshes.values.forEach { mesh ->
mesh.push.primeBufferReferences(cmd) {
throw MissingRefException(afield.graphics.vulkan.buffers.unassignedMemberMessage(it))
}
}
mp.current.liveUniformStructures.values.forEach { structure ->
structure.primeBufferReferences(cmd) {
throw MissingRefException(afield.graphics.vulkan.buffers.unassignedMemberMessage(it))
}
}
}
modules.forEach { (_, mp) ->
mp.current.before.forEach { (_, target) ->
target.render(cmd, 0)
}
}
meshModules.forEach { (_, mp) ->
mp.current.before.forEach { (_, target) ->
target.render(cmd, 0)
}
}
// Texture providers may need layout transitions before descriptors are pushed.
// Dynamic rendering does not allow image barriers inside the rendering instance.
modules.forEach { (_, mp) -> mp.prepareSampledImagesFor(cmd) }
meshModules.forEach { (_, mp) -> mp.prepareSampledImagesFor(cmd) }
cmd.inRenderPassInstance = true
KHRDynamicRendering.vkCmdBeginRenderingKHR(cmd.vkCommandBuffer, renderInfo)
// Render all modules into this eye
val fullRect = sc.fullRect
val rectF = sc.rectF
modules.forEach { (_, mp) ->
MemoryStack.stackPush().use { s2 ->
setViewportAndScissor(cmd.vkCommandBuffer, mp.current.parameters.viewport ?: fullRect, s2)
}
mp.render(cmd, rectF)
}
meshModules.forEach { (_, mp) ->
MemoryStack.stackPush().use { s2 ->
setViewportAndScissor(cmd.vkCommandBuffer, mp.current.parameters.viewport ?: fullRect, s2)
}
mp.render(cmd, rectF)
}
KHRDynamicRendering.vkCmdEndRenderingKHR(cmd.vkCommandBuffer)
cmd.inRenderPassInstance = false
cmd.flushDeferredGraphicsHostReadBarriers()
// End per-eye GPU timing
ps?.let { p ->
synchronized(p.lock) {
eyeMarks.tsEnd = p.profiler.writeTimestamp2(
cmd.vkCommandBuffer, 0, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT
)
p.perFrameEyeMarks.add(eyeMarks)
}
}
}
}
}
// End GPU profiling
ps?.let { p ->
synchronized(p.lock) {
p.profiler.endPerfCounters(cmd.vkCommandBuffer, 0, perfScope)
perfScope?.let { p.markers.perfFrameIndex = it.index }
p.profiler.endPipelineStats(cmd.vkCommandBuffer, eyesScope)
p.markers.tsAfterEyes = p.profiler.writeTimestamp2(
cmd.vkCommandBuffer, 0, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT
)
p.markers.tsFrameEnd = p.profiler.writeTimestamp2(
cmd.vkCommandBuffer, 0, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT
)
}
}
currentEyeIndex = -1
cmd.endRecording()
// Submit and wait for completion so we can safely xrReleaseSwapchainImage.
MemoryStack.stackPush().use { stack ->
renderer.graphQueue.submit(
stack.pointers(cmd.vkCommandBuffer),
null,
null,
null,
fence
)
}
// No wait here: we either wait just before release (conservative), or defer the wait until
// the next frame when we need to reuse the command buffer/fence.
}
private fun pollEvents(): Boolean {
val xrInstanceLocal = xrInstance ?: return false
val xrSessionLocal = xrSession
MemoryStack.stackPush().use { stack ->
val eventData = XrEventDataBuffer.calloc(stack)
while (true) {
eventData.type(XR_TYPE_EVENT_DATA_BUFFER)
val res = xrPollEvent(xrInstanceLocal, eventData)
if (res == XR_EVENT_UNAVAILABLE) break
if (res != XR_SUCCESS) return false
when (eventData.type()) {
XR_TYPE_EVENT_DATA_SESSION_STATE_CHANGED -> {
val changed = XrEventDataSessionStateChanged.create(eventData.address())
sessionState = changed.state()
when (sessionState) {
XR_SESSION_STATE_READY -> {
if (xrSessionLocal != null) {
val beginInfo = XrSessionBeginInfo.calloc(stack)
.type(XR_TYPE_SESSION_BEGIN_INFO)
.primaryViewConfigurationType(XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO)
val br = xrBeginSession(xrSessionLocal, beginInfo)
if (br == XR_SUCCESS) sessionRunning = true
}
}
XR_SESSION_STATE_STOPPING -> {
if (xrSessionLocal != null) {
xrEndSession(xrSessionLocal)
}
sessionRunning = false
}
XR_SESSION_STATE_EXITING, XR_SESSION_STATE_LOSS_PENDING -> return false
}
}
XR_TYPE_EVENT_DATA_INSTANCE_LOSS_PENDING -> return false
}
}
}
return true
}
private fun setViewportAndScissor(cmd: VkCommandBuffer, rect: Rect, stack: MemoryStack) {
val viewport = VkViewport.calloc(1, stack)
.x(rect.x.toFloat())
.y((rect.y + rect.h).toFloat())
.height((-rect.h).toFloat())
.width(rect.w.toFloat())
.minDepth(0f)
.maxDepth(1f)
vkCmdSetViewport(cmd, 0, viewport)
val scissor = VkRect2D.calloc(1, stack)
.extent(
VkExtent2D.calloc(stack)
.width(rect.w.toInt())
.height(rect.h.toInt())
)
.offset(VkOffset2D.calloc(stack).x(rect.x.toInt()).y(rect.y.toInt()))
vkCmdSetScissor(cmd, 0, scissor)
}
private fun resolveProfilingForFrame(@Suppress("UNUSED_PARAMETER") frameCounter: Long) {
val ps = profilerState ?: return
synchronized(ps.lock) {
val p = ps.profiler
val m = ps.markers
// Print performance counter data
if (m.perfFrameIndex >= 0 && p.opts.perfCounters) {
val meta = p.perfMeta
val vals = p.fetchPerfCounters(0, m.perfFrameIndex, meta.size)
val n = kotlin.math.min(vals.size, meta.size)
if (n > 0) {
val byScope = meta.withIndex().groupBy { it.value.scope }
for ((scope, items) in byScope) {
lprintln("info,perf", "XR scope=$scope")
for (e in items) {
val i = e.index
lprintln("info,perf", " ${e.value.name}: ${p.formatPerfValue(vals[i], e.value.unit)}")
}
}
}
}
// Print frame timing data
if (m.tsFrameBegin >= 0 && m.tsFrameEnd >= 0) {
val ts = p.fetchTimestampNsRange(0, m.tsFrameBegin, m.tsFrameEnd)
if (ts.size >= 3) {
val frameNs = ts[2] - ts[0]
val eyesNs = ts[1] - ts[0]
val postNs = ts[2] - ts[1]
lprintln(
"info,perf",
"XR GPU frame=${frameNs/1e6} ms (eyes=${eyesNs/1e6} ms, post=${postNs/1e6} ms)"
)
}
}
// Print pipeline stats
if (m.pipeStatsEyes >= 0 && p.opts.pipelineStats) {
val stats = p.fetchPipelineStats(0, m.pipeStatsEyes)
val iaVerts = stats.getOrNull(0) ?: -1
val iaPrims = stats.getOrNull(1) ?: -1
val vsInv = stats.getOrNull(2) ?: -1
val fsInv = stats.getOrNull(7) ?: -1
lprintln("info,perf", "XR Eyes IA verts=$iaVerts, IA prims=$iaPrims, VS invoc=$vsInv, FS invoc=$fsInv")
}
// Print per-eye timing
val eyeMarks = ps.perFrameEyeMarks
if (eyeMarks.isNotEmpty()) {
for (em in eyeMarks) {
if (em.tsBegin >= 0 && em.tsEnd >= 0) {
val ts = p.fetchTimestampNsRange(0, em.tsBegin, em.tsEnd)
if (ts.size >= 2) {
val dtMs = (ts.last() - ts.first()) / 1e6
lprintln("info,perf", "XR Eye[${em.eyeIndex}]: ${"%.3f".format(dtMs)} ms")
}
}
}
}
}
}
fun cleanup()
body — 86 lines, from line 2912
fun cleanup() {
renderer.runOnQueueThread {
if (initialized) {
fence.fenceWait()
}
}
// Profiling resources
profilerState?.profiler?.destroy()
profilerState = null
// Vulkan resources
swapchains.forEach { sc ->
sc.depthAttachments.forEach { it.cleanup() }
sc.imageViews.forEach { vkDestroyImageView(renderer.device.vkDevice, it, null) }
xrDestroySwapchain(sc.swapchain)
}
swapchains.clear()
// XR resources
eyeGazeInteraction?.close()
eyeGazeInteraction = null
htcEyeTracker?.close()
htcEyeTracker = null
leftHandTracker?.let { EXTHandTracking.xrDestroyHandTrackerEXT(it) }
rightHandTracker?.let { EXTHandTracking.xrDestroyHandTrackerEXT(it) }
leftHandTracker = null
rightHandTracker = null
leftJointsBuf?.free(); leftJointsBuf = null
rightJointsBuf?.free(); rightJointsBuf = null
handTrackingEnabled = false
bodyTrackers.forEach { xrDestroySpace(it.space) }
bodyTrackers = emptyList()
if (xdevList != 0L) xdevSpaceExt?.destroyXDevList(xdevList)
xdevList = 0L
xdevGeneration = -1L
xdevSpaceExt = null
bodyJointMapping = emptyList()
bodyTrackingEnabled = false
leftGripSpace?.let { xrDestroySpace(it) }
rightGripSpace?.let { xrDestroySpace(it) }
leftAimSpace?.let { xrDestroySpace(it) }
rightAimSpace?.let { xrDestroySpace(it) }
xrActionSet?.let { xrDestroyActionSet(it) }
xrSpace?.let { xrDestroySpace(it) }
xrSession?.let { xrDestroySession(it) }
xrInstance?.let { xrDestroyInstance(it) }
leftGripSpace = null
rightGripSpace = null
leftAimSpace = null
rightAimSpace = null
xrActionSet = null
xrPoseAction = null
xrAimPoseAction = null
xrSelectAction = null
xrTriggerValueAction = null
xrTriggerClickAction = null
xrSqueezeValueAction = null
xrSqueezeClickAction = null
xrTrackpadAction = null
xrTrackpadTouchAction = null
xrTrackpadClickAction = null
xrThumbstickAction = null
xrThumbstickTouchAction = null
xrThumbstickClickAction = null
xrAAction = null
xrBAction = null
xrMenuAction = null
xrHapticAction = null
xrSpace = null
xrSession = null
xrInstance = null
viewConfigs?.free(); viewConfigs = null
// Vulkan submission resources
fence.cleanup()
cmd.cleanup()
disabledReason = "cleaned up"
}
}