WebcamTexture.kt

    override fun browsable(): Browsable? = object : Browsable {
        override fun heading(): String = "WebcamTexture"
        override fun dir(): Map<String, Any?>? = null
        override fun html(): String {
            fun esc(s: String) = s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
            val snapshot = synchronized(stateLock) {
                mapOf(
                    "device" to deviceIndex.toString(),
                    "size" to if (cameraWidth > 0) "${cameraWidth}x${cameraHeight}" else "(uninitialized)",
                    "state" to if (running) "running" else "stopped",
                    "captured" to capturedFrames.toString(),
                    "uploaded" to uploadedFrames.toString(),
                    "dropped" to droppedFrames.toString(),
                    "error" to (lastError?.message ?: "")
                )
            }
            return """
                <div style="font-family:inherit;color:#ddd;padding:6px 8px;">
                  <div style="display:grid;grid-template-columns:120px 1fr;row-gap:4px;column-gap:10px;align-items:baseline;">
                    <div style="text-align:right;color:#8a8a8a;font-size:12px;">device</div><div>${snapshot["device"]}</div>
                    <div style="text-align:right;color:#8a8a8a;font-size:12px;">size</div><div>${snapshot["size"]}</div>
                    <div style="text-align:right;color:#8a8a8a;font-size:12px;">state</div><div>${snapshot["state"]}</div>
                    <div style="text-align:right;color:#8a8a8a;font-size:12px;">captured</div><div>${snapshot["captured"]}</div>
                    <div style="text-align:right;color:#8a8a8a;font-size:12px;">uploaded</div><div>${snapshot["uploaded"]}</div>
                    <div style="text-align:right;color:#8a8a8a;font-size:12px;">dropped</div><div>${snapshot["dropped"]}</div>
                    <div style="text-align:right;color:#8a8a8a;font-size:12px;">error</div><div style="white-space:pre-wrap;">${esc(snapshot["error"] ?: "")}</div>
                  </div>
                </div>
            """.trimIndent()
        }

        override fun version(): Long = synchronized(stateLock) { latestFrameIdInternal + (uploadedFrameId shl 32) }
    }

    private fun ensureInitialized(): WebcamRgbaTexture {
        texture?.let { return it }

        synchronized(stateLock) {
            texture?.let { return it }

            val newGrabber = OpenCVFrameGrabber(deviceIndex).apply {
                requestedWidth?.let { imageWidth = it }
                requestedHeight?.let { imageHeight = it }
                requestedFps?.let { frameRate = it }
            }

            try {
                newGrabber.start()

                val firstFrame = awaitFirstFrame(newGrabber)
                val firstRgba = convertFrameToRgba(firstFrame)
                val width = firstRgba.cols()
                val height = firstRgba.rows()
                val bytes = width * height * 4

                val latestBuffer = MemoryUtil.memAlloc(bytes)
                copyMatToBuffer(firstRgba, latestBuffer, bytes)

                val newTexture = WebcamRgbaTexture(
                    renderer = renderer,
                    width = width,
                    height = height,
                    name = "webcam_$deviceIndex"
                )
                newTexture.uploadNow(latestBuffer, bytes)

                grabber = newGrabber
                texture = newTexture
                sharedFrameBuffer = latestBuffer
                cameraWidth = width
                cameraHeight = height
                frameBytes = bytes
                latestFrameIdInternal = 1L
                uploadedFrameId = 1L
                capturedFrames = 1L
                uploadedFrames = 1L
                droppedFrames = 0L
                warnedAboutResize = false
                lastError = null
                running = true

                startCaptureThreadLocked()
                return newTexture
            } catch (t: Throwable) {
                lastError = t
                runCatching { newGrabber.stop() }
                runCatching { newGrabber.release() }
                throw IllegalStateException("Failed to initialize webcam $deviceIndex", t)
            }
        }
    }

    private fun startCaptureThreadLocked() {
        if (captureThread?.isAlive == true) return

        val thread = Thread({
            captureLoop()
        }, "WebcamTexture-$deviceIndex").apply {
            isDaemon = true
        }
        captureThread = thread
        thread.start()
    }

    private fun captureLoop() {
        while (true) {
            val activeGrabber = synchronized(stateLock) {
                if (!running) return
                grabber
            } ?: return

            try {
                val frame = activeGrabber.grab() ?: continue
                val rgba = convertFrameToRgba(frame)
                val width = rgba.cols()
                val height = rgba.rows()

                synchronized(stateLock) {
                    if (!running) return

                    if (width != cameraWidth || height != cameraHeight) {
                        if (!warnedAboutResize) {
                            warnedAboutResize = true
                            lprintln(
                                "warning,webcam",
                                "Ignoring webcam resize ${width}x${height}; texture remains ${cameraWidth}x${cameraHeight}"
                            )
                        }
                        return@synchronized
                    }

                    val latestBuffer = sharedFrameBuffer ?: return@synchronized
                    copyMatToBuffer(rgba, latestBuffer, frameBytes)
                    if (latestFrameIdInternal != uploadedFrameId) {
                        droppedFrames++
                    }
                    latestFrameIdInternal++
                    capturedFrames++
                }
            } catch (_: InterruptedException) {
                return
            } catch (t: Throwable) {
                lastError = t
                lprintln("error,webcam", "Capture loop failed for webcam $deviceIndex: ${t.message}")
                synchronized(stateLock) {
                    running = false
                }
                return
            }
        }
    }

    private fun awaitFirstFrame(grabber: OpenCVFrameGrabber): Frame {
        val deadline = System.nanoTime() + startupTimeoutMs * 1_000_000L
        while (System.nanoTime() < deadline) {
            val frame = grabber.grab()
            if (frame != null) {
                return frame
            }
            Thread.sleep(10L)
        }
        throw IllegalStateException("Timed out waiting for webcam $deviceIndex to produce a frame")
    }

    private fun convertFrameToRgba(frame: Frame): Mat {
        val source = converter.convert(frame)
            ?: throw IllegalStateException("OpenCVFrameGrabber returned a frame that could not be converted to Mat")

        val width = source.cols()
        val height = source.rows()
        val target = ensureScratch(width, height)

        when (source.channels()) {
            4 -> cvtColor(source, target, COLOR_BGRA2RGBA)
            3 -> cvtColor(source, target, COLOR_BGR2RGBA)
            1 -> cvtColor(source, target, COLOR_GRAY2RGBA)
            else -> throw IllegalStateException("Unsupported webcam channel count: ${source.channels()}")
        }
        return target
    }

    private fun ensureScratch(width: Int, height: Int): Mat {
        val current = rgbaScratch
        if (current != null && current.cols() == width && current.rows() == height) {
            return current
        }

        current?.release()
        current?.close()

        return Mat(height, width, CV_8UC4).also {
            rgbaScratch = it
        }
    }

    private fun copyMatToBuffer(mat: Mat, destination: java.nio.ByteBuffer, bytes: Int) {
        val srcPointer = mat.data()
            ?: throw IllegalStateException("Converted webcam frame had no backing pixel buffer")
        val src = MemoryUtil.memByteBuffer(srcPointer.address(), bytes)
        val srcCopy = src.duplicate()
        srcCopy.position(0)
        srcCopy.limit(bytes)

        destination.clear()
        destination.put(srcCopy)
        destination.flip()
    }
}

private class WebcamRgbaTexture(
    private val renderer: Renderer,
    val width: Int,
    val height: Int,
    name: String
) : HasImageView {

    private val image: Image
    override val imageView: ImageView
    private val stagingBuffer: VulkanBuffer
    private var currentLayout = VK10.VK_IMAGE_LAYOUT_UNDEFINED

    init {
        val imageBytes = width * height * 4L
        stagingBuffer = VulkanBuffer(
            renderer,
            imageBytes,
            VK10.VK_BUFFER_USAGE_TRANSFER_SRC_BIT,
            VK10.VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT or VK10.VK_MEMORY_PROPERTY_HOST_COHERENT_BIT
        )

        image = Image(
            renderer.device,
            Image.ImageData()
                .width(width)
                .height(height)
                .format(VK10.VK_FORMAT_R8G8B8A8_UNORM)
                .usage(VK10.VK_IMAGE_USAGE_TRANSFER_DST_BIT or VK10.VK_IMAGE_USAGE_SAMPLED_BIT)
                .mipLevels(1)
        )

        imageView = ImageView(
            renderer.device,
            image,
            ImageView.ImageViewData()
                .format(VK10.VK_FORMAT_R8G8B8A8_UNORM)
                .aspectMask(VK10.VK_IMAGE_ASPECT_COLOR_BIT)
                .mipLevels(1)
        )
    }

    fun upload(cmd: CommandBuffer, source: java.nio.ByteBuffer, bytes: Int) {
        copyToStaging(source, bytes)
        recordUpload(cmd)
    }

    fun uploadNow(source: java.nio.ByteBuffer, bytes: Int) {
        val cmd = CommandBuffer(renderer.commandPool, true, true)
        cmd.beginRecording()
        upload(cmd, source, bytes)
        cmd.endRecording()

        val fence = Fence(renderer.device, true)
        fence.reset()
        MemoryStack.stackPush().use { stack ->
            renderer.graphQueue.submit(stack.pointers(cmd.vkCommandBuffer), null, null, null, fence)
        }
        fence.fenceWait()
        fence.cleanup()
        cmd.cleanup()
    }

    fun cleanup() {
        stagingBuffer.cleanup()
        imageView.cleanup()
        image.cleanup()
    }

    private fun copyToStaging(source: java.nio.ByteBuffer, bytes: Int) {
        val src = source.duplicate()
        src.position(0)
        src.limit(bytes)

        val mapped = stagingBuffer.map()
        val dst = MemoryUtil.memByteBuffer(mapped, bytes)
        dst.position(0)
        dst.put(src)
        stagingBuffer.unmap(0, bytes.toLong())
    }

    private fun recordUpload(cmd: CommandBuffer) {
        MemoryStack.stackPush().use { stack ->
            recordImageTransition(stack, cmd, currentLayout, VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL)
            recordCopyBuffer(stack, cmd)
            recordImageTransition(
                stack,
                cmd,
                VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
                VK10.VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL
            )
        }
        currentLayout = VK10.VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL
    }

    private fun recordCopyBuffer(stack: MemoryStack, cmd: CommandBuffer) {
        val region = VkBufferImageCopy.calloc(1, stack)
            .bufferOffset(0)
            .bufferRowLength(0)
            .bufferImageHeight(0)
            .imageSubresource { it: VkImageSubresourceLayers ->
                it.aspectMask(VK10.VK_IMAGE_ASPECT_COLOR_BIT)
                    .mipLevel(0)
                    .baseArrayLayer(0)
                    .layerCount(1)
            }
            .imageOffset { it: VkOffset3D -> it.x(0).y(0).z(0) }
            .imageExtent { it: VkExtent3D ->
                it.width(width).height(height).depth(1)
            }

        VK10.vkCmdCopyBufferToImage(
            cmd.vkCommandBuffer,
            stagingBuffer.buffer,
            image.vkImage,
            vkLayout(VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL),
            region
        )
    }

    private fun recordImageTransition(stack: MemoryStack, cmd: CommandBuffer, oldLayout: Int, newLayout: Int) {
        val barrier = VkImageMemoryBarrier.calloc(1, stack)
            .sType(VK10.VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER)
            .oldLayout(vkLayout(oldLayout))
            .newLayout(vkLayout(newLayout))
            .srcQueueFamilyIndex(VK10.VK_QUEUE_FAMILY_IGNORED)
            .dstQueueFamilyIndex(VK10.VK_QUEUE_FAMILY_IGNORED)
            .image(image.vkImage)
            .subresourceRange { it: VkImageSubresourceRange ->
                it.aspectMask(VK10.VK_IMAGE_ASPECT_COLOR_BIT)
                    .baseMipLevel(0)
                    .levelCount(1)
                    .baseArrayLayer(0)
                    .layerCount(1)
            }

        val srcStage: Int
        val srcAccessMask: Int
        val dstAccessMask: Int
        val dstStage: Int

        when {
            oldLayout == VK10.VK_IMAGE_LAYOUT_UNDEFINED && newLayout == VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL -> {
                srcStage = VK10.VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT
                srcAccessMask = 0
                dstStage = VK10.VK_PIPELINE_STAGE_TRANSFER_BIT
                dstAccessMask = VK10.VK_ACCESS_TRANSFER_WRITE_BIT
            }

            oldLayout == VK10.VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL && newLayout == VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL -> {
                srcStage = VK10.VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT
                srcAccessMask = VK10.VK_ACCESS_SHADER_READ_BIT
                dstStage = VK10.VK_PIPELINE_STAGE_TRANSFER_BIT
                dstAccessMask = VK10.VK_ACCESS_TRANSFER_WRITE_BIT
            }

            oldLayout == VK10.VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL && newLayout == VK10.VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL -> {
                srcStage = VK10.VK_PIPELINE_STAGE_TRANSFER_BIT
                srcAccessMask = VK10.VK_ACCESS_TRANSFER_WRITE_BIT
                dstStage = VK10.VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT
                dstAccessMask = VK10.VK_ACCESS_SHADER_READ_BIT
            }

            else -> throw IllegalStateException("Unsupported image layout transition: $oldLayout -> $newLayout")
        }

        barrier.srcAccessMask(srcAccessMask)
        barrier.dstAccessMask(dstAccessMask)

        VK10.vkCmdPipelineBarrier(
            cmd.vkCommandBuffer,
            srcStage,
            dstStage,
            0,
            null,
            null,
            barrier
        )
    }
}