PortableOpticalFlowSmoke.kt

    @JvmStatic fun main(args: Array<String>) {
        Options.enableVR=false;Options.enableHDR=false;Options.enableRayTracing=false;Options.doVulkanSDK()
        val dir=File("build/portable-optical-flow").also { it.mkdirs() }
        val window=Window("Field portable optical flow");val renderer=Renderer(window);val box=Box("portable-flow-check")
        box.setPropertyRaw("renderer",renderer,doResync=false)
        val report=JSONObject()
        val failures=mutableListOf<String>()
        fun expect(ok: Boolean,message: ()->String) { if(!ok)failures+=message() }
        try { Box.withBoxContextNow(box) {
            report.put("device",renderer.device.physicalDevice.vkPhysicalDeviceProperties.deviceNameString())
            report.put("autoBackend",OpticalFlowBackend.AUTO.resolve(renderer).name)
            OpticalFlowDemo(renderer,backend=OpticalFlowBackend.PYRAMID).use { demo ->
                val flow=demo.flow
                check(flow is PyramidOpticalFlow && flow.backend==OpticalFlowBackend.PYRAMID)
                check(flow.flowWidth==(demo.width+3)/4 && flow.flowHeight==(demo.height+3)/4) { "Output dimensions break the grid contract" }
                report.put("input",listOf(demo.width,demo.height)).put("grid",flow.grid).put("levels",flow.levels)
                    .put("radius",flow.radius).put("iterations",flow.iterations).put("finestIterations",flow.finestIterations).put("allocatedBytes",flow.allocatedBytes)

                // Premise: the pair is textured, differs, and the second is exactly the first moved by (dx, dy).
                demo.update(reset=true)
                val a=ReconstructionCapture.read(renderer,demo.pair.imageViewArray[0]);val b=ReconstructionCapture.read(renderer,demo.pair.imageViewArray[1])
                fun grey(p: FloatArray,x: Int,y: Int)=((y*demo.width+x)*4).let { .2126f*p[it]+.7152f*p[it+1]+.0722f*p[it+2] }
                var sum=0.0;var sumSq=0.0;var unmoved=0.0;var moved=0.0;var n=0
                for(y in 64 until demo.height-64)for(x in 64 until demo.width-64) {
                    val g=grey(a,x,y);sum+=g;sumSq+=g*g;n++
                    unmoved+=abs(grey(b,x,y)-g)
                    moved+=abs(grey(b,x+demo.dx.toInt(),y+demo.dy.toInt())-g)
                }
                val premise=JSONObject().put("greyStd",sqrt(sumSq/n-(sum/n)*(sum/n))).put("meanAbsDifferenceUnmoved",unmoved/n)
                    .put("meanAbsDifferenceAfterKnownShift",moved/n)
                report.put("premise",premise);println("PREMISE $premise")
                check(premise.getDouble("greyStd")>.05 && unmoved/n>.02) { "Source pair is too flat or too similar to test flow: $premise" }
                check(moved/n<1e-3) { "Second image is not the first moved by (${demo.dx}, ${demo.dy}): $premise" }

                fun errors(time: Float): List<Float> {
                    val field=ReconstructionCapture.read(renderer,flow.output)
                    check(field.all { it.isFinite() }) { "Non-finite flow" }
                    val out=ArrayList<Float>()
                    for(y in 16 until flow.flowHeight-16)for(x in 16 until flow.flowWidth-16) {
                        val i=(y*flow.flowWidth+x)*4
                        val e=demo.expectedDisplacement((x+.5f)*flow.grid,(y+.5f)*flow.grid,time)
                        out+=sqrt((field[i]-e.x)*(field[i]-e.x)+(field[i+1]-e.y)*(field[i+1]-e.y))
                    }
                    return out
                }

                // Known translation (+12, -8), through the demo's frame loop.
                repeat(12){demo.update(reset=it==0);window.pollEvents();renderer.render()}
                val translation=summary(errors(0f));report.put("translation",translation);println("TRANSLATION $translation")
                // Cost: flow is first -> second, so first-image content in the rightmost 12 px (dx = +12) leaves the
                // frame and has no correct match. The two rightmost flow columns (8 px) are wholly inside that band;
                // their residual should stand clear of the interior's, and above the demo's default reject threshold (8).
                run {
                    val field=ReconstructionCapture.read(renderer,flow.output);val leaving=ArrayList<Float>();val interior=ArrayList<Float>()
                    for(y in 16 until flow.flowHeight-16)for(x in 16 until flow.flowWidth){val z=field[(y*flow.flowWidth+x)*4+2]
                        if(x>=flow.flowWidth-2)leaving+=z else if(x<flow.flowWidth-16)interior+=z}
                    val cost=JSONObject().put("units","mean absolute window residual, grey levels 0-255")
                        .put("leavingColumns",summary(leaving)).put("interior",summary(interior))
                    report.put("translationCost",cost);println("COST $cost")
                    expect(percentile(leaving.sorted(),.5)>maxOf(8f,2*percentile(interior.sorted(),.95))) { "Cost does not flag content leaving the frame: $cost" }
                }
                ReconstructionCapture.save(File(dir,"translation.png"),ReconstructionCapture.read(renderer,demo.output),demo.width*2,demo.height*2)
                expect(translation.getDouble("median")<TRANSLATION_MEDIAN && translation.getDouble("p95")<TRANSLATION_P95) { "Translation: $translation" }

                // A displacement beyond one window at full resolution: only the pyramid can recover it.
                demo.dx=40f;demo.dy=-24f;demo.update(reset=true)
                val large=summary(errors(0f));report.put("largeTranslation40x24",large);println("LARGE $large")
                expect(large.getDouble("median")<LARGE_MEDIAN && large.getDouble("p95")<LARGE_P95 &&
                    large.getDouble("fractionOver1px")<LARGE_OUTLIER_FRACTION) { "Large translation: $large" }

                // Other grids, on the same pair, through the public factory.
                demo.dx=12f;demo.dy=-8f;demo.update(reset=true)
                val grids=JSONObject()
                // AUTO must pick the class that matches this device.
                opticalFlow(renderer,demo.pair.imageViewArray[0],demo.pair.imageViewArray[1]).use { auto ->
                    check((auto is HardwareOpticalFlow)==renderer.device.supportsOpticalFlow && (auto is PyramidOpticalFlow)!=renderer.device.supportsOpticalFlow) { "AUTO chose ${auto.backend}" }
                }
                for(g in listOf(2,8)) opticalFlow(renderer,demo.pair.imageViewArray[0],demo.pair.imageViewArray[1],grid=g,backend=OpticalFlowBackend.PYRAMID).use { other ->
                    check(other is PyramidOpticalFlow)
                    other.evaluate(true)
                    check(other.flowWidth==(demo.width+g-1)/g && other.flowHeight==(demo.height+g-1)/g)
                    val field=ReconstructionCapture.read(renderer,other.output);val margin=64/g;val e=ArrayList<Float>()
                    for(y in margin until other.flowHeight-margin)for(x in margin until other.flowWidth-margin){val i=(y*other.flowWidth+x)*4
                        e+=sqrt((field[i]-12)*(field[i]-12)+(field[i+1]+8)*(field[i+1]+8))}
                    grids.put("grid$g",summary(e))
                    expect(percentile(e.sorted(),.5)<TRANSLATION_MEDIAN && percentile(e.sorted(),.95)<TRANSLATION_P95) { "Grid $g translation: ${summary(e)}" }
                }
                report.put("translationOtherGrids",grids);println("GRIDS $grids")

                // Forward–backward consistency on the same (+12, -8) pair: near zero where both directions agree,
                // large where first-image content leaves the frame (rightmost 12 px) and has no backward partner.
                PyramidOpticalFlow(renderer,demo.pair.imageViewArray[0],demo.pair.imageViewArray[1],consistency=true).use { checked ->
                    checked.evaluate(true)
                    val map=ReconstructionCapture.read(renderer,checked.consistencyMap!!);val w=checked.flowWidth
                    val leaving=ArrayList<Float>();val interior=ArrayList<Float>()
                    for(y in 16 until checked.flowHeight-16)for(x in 16 until w){val e=map[(y*w+x)*4]
                        if(x>=w-2)leaving+=e else if(x<w-16)interior+=e}
                    repeat(20){checked.evaluate()}
                    val times=checked.milliseconds.takeLast(20).sorted()
                    val fb=JSONObject().put("units","forward-backward endpoint error, input pixels").put("interior",summary(interior))
                        .put("leavingColumns",summary(leaving)).put("pairWallMsP50",times[times.size/2])
                    report.put("translationConsistency",fb);println("CONSISTENCY $fb")
                    expect(percentile(interior.sorted(),.95)<CONSISTENCY_INTERIOR_P95 && percentile(leaving.sorted(),.5)>CONSISTENCY_LEAVING_MEDIAN) { "Consistency: $fb" }
                }

                // Timing: 100 warmed pairs.
                repeat(100){demo.update();window.pollEvents();renderer.render()}
                val times=flow.milliseconds.takeLast(100).sorted()
                report.put("pairWallMsIncludingFence",JSONObject().put("count",times.size).put("p50",percentile(times.map{it.toFloat()},.5)).put("p95",percentile(times.map{it.toFloat()},.95)))
                flow.gpuTimings.forEach { (name,values) -> val s=values.takeLast(100).sorted()
                    report.put("gpuMs_"+name,if(s.isEmpty())JSONObject.NULL else JSONObject().put("count",s.size).put("p50",s[s.size/2]).put("p95",s[((s.size-1)*.95).toInt()])) }

                // The page's consecutive-frame wiggle: a local warp, no global offset, at four phases.
                demo.deformation=true;demo.dx=0f;demo.dy=0f;demo.displayRange=3f
                val local=ArrayList<Float>();val zeroField=ArrayList<Float>();val quadrants=IntArray(4)
                for(time in listOf(0f,.4f,.9f,1.5f)) {
                    demo.update(time,reset=true);local+=errors(time)
                    for(y in 16 until flow.flowHeight-16)for(x in 16 until flow.flowWidth-16)
                        zeroField+=demo.expectedDisplacement((x+.5f)*flow.grid,(y+.5f)*flow.grid,time).let { sqrt(it.x*it.x+it.y*it.y) }
                    val field=ReconstructionCapture.read(renderer,flow.output)
                    for(y in 16 until flow.flowHeight-16)for(x in 16 until flow.flowWidth-16){val i=(y*flow.flowWidth+x)*4
                        if(abs(field[i])>.2 && abs(field[i+1])>.2)quadrants[(if(field[i]>0)1 else 0)+(if(field[i+1]>0)2 else 0)]++}
                }
                // What a zero field would score: the premise that the wiggle is motion the floors can tell apart.
                val zero=summary(zeroField);check(zero.getDouble("median")>5*WIGGLE_MEDIAN) { "Wiggle too small to test: $zero" }
                val wiggleCost=ArrayList<Float>();ReconstructionCapture.read(renderer,flow.output).let { f ->
                    for(y in 16 until flow.flowHeight-16)for(x in 16 until flow.flowWidth-16)wiggleCost+=f[(y*flow.flowWidth+x)*4+2] }
                val wiggle=summary(local).put("zeroFieldError",zero).put("phases",4).put("costLastPhase",summary(wiggleCost)).put("pairStepSeconds",demo.pairStep).put("directionQuadrants",quadrants.toList())
                report.put("consecutiveFrameWiggle",wiggle);println("WIGGLE $wiggle")
                expect(wiggle.getDouble("median")<WIGGLE_MEDIAN && wiggle.getDouble("p95")<WIGGLE_P95) { "Wiggle: $wiggle" }
                expect(quadrants.all { it>local.size/20 }) { "Wiggle field did not change direction: ${quadrants.toList()}" }
                // Consistency on the wiggle: both directions track a smooth warp, so the check should mostly agree.
                PyramidOpticalFlow(renderer,demo.pair.imageViewArray[0],demo.pair.imageViewArray[1],consistency=true).use { checked ->
                    checked.evaluate(true);val map=ReconstructionCapture.read(renderer,checked.consistencyMap!!);val e=ArrayList<Float>()
                    for(y in 16 until checked.flowHeight-16)for(x in 16 until checked.flowWidth-16)e+=map[(y*checked.flowWidth+x)*4]
                    val fb=summary(e);report.put("wiggleConsistencyLastPhase",fb);println("WIGGLE CONSISTENCY $fb")
                    expect(percentile(e.sorted(),.95)<WIGGLE_CONSISTENCY_P95) { "Wiggle consistency: $fb" }
                }
                repeat(8){i->demo.update(1.5f+i*demo.pairStep);window.pollEvents();renderer.render()}
                ReconstructionCapture.save(File(dir,"wiggle.png"),ReconstructionCapture.read(renderer,demo.output),demo.width*2,demo.height*2)

                // Identical frames: no motion anywhere, borders included.
                demo.pairStep=0f;demo.update(.5f,reset=true)
                val still=ReconstructionCapture.read(renderer,flow.output)
                val stillLengths=(0 until flow.flowWidth*flow.flowHeight).map { sqrt(still[it*4]*still[it*4]+still[it*4+1]*still[it*4+1]) }
                val stationary=summary(stillLengths);report.put("identicalFrames",stationary);println("IDENTICAL $stationary")
                expect(stationary.getDouble("max")<IDENTICAL_MAX) { "Identical frames produced motion: $stationary" }

                // Feedback advection driven by the measured field, as the starting point shows it.
                demo.pairStep=1f/30f
                repeat(if("--clip" in args)120 else 32){i ->demo.update(i*demo.pairStep,reset=i==0);window.pollEvents();renderer.render()
                    if("--clip" in args)ReconstructionCapture.save(File(dir,"clip/%04d.png".format(i)),ReconstructionCapture.read(renderer,demo.output),demo.width*2,demo.height*2)}
                ReconstructionCapture.save(File(dir,"advection.png"),ReconstructionCapture.read(renderer,demo.output),demo.width*2,demo.height*2)
            }
            // A stopped parent renderer must also retire an otherwise still-reachable flow view.
            OpticalFlowDemo(renderer,128,128,name="unclosed-portable-flow-check",backend=OpticalFlowBackend.PYRAMID).update(reset=true)
        } } finally {renderer.cleanup();window.cleanup()}
        report.put("failures",failures)
        File(dir,"measurements.json").writeText(report.toString(2))
        if(failures.isNotEmpty()) { failures.forEach { System.err.println("FAIL $it") };exitProcess(1) }
        println("PORTABLE OPTICAL FLOW CHECK PASSED");exitProcess(0)
    }