CorpusOperations.kt

src/jvmMain/kotlin/afield/audio/ai/CorpusOperations.kt · 153 lines

@file:afield.runtime.completion.CompletionCategory("Audio AI")

package afield.audio.ai

import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonArray
import kotlinx.serialization.json.putJsonObject
import kotlin.math.abs

Multi-scale corpus operations: segment a source into overlapping windows, embed each window with grouped audio descriptors, and retrieve neighbours by weighted distance.

The three operations form a pipeline that runs server-side. segmentCorpus enumerates deterministic time windows at several scales without computing any descriptors. embedCorpus computes and persists the five grouped descriptor families (harmony, pitch, texture, rhythm, dynamics) for every window. findCorpusNeighbours retrieves the closest embedded windows to a query using CorpusDistanceWeights, which controls how much each group contributes to the distance. All three produce a ModelOperation that FieldAudioClient.submit executes.

data class CorpusDistanceWeights

Explicit weights for the five interpretable descriptor groups used by corpus retrieval. Each scales how much its own group counts towards the distance between two windows, a zero takes that group out of the comparison, and at least one of the five has to be positive.

data class CorpusDistanceWeights(
    /** Mean and variation of the twelve chroma classes: the chord and key colour of a window. */
    val harmony: Double = 1.0,
    /** Dominant-pitch centre, dominant-pitch spread, and how much of the window is voiced. */
    val pitch: Double = 0.7,
    /** MFCCs with spectral centroid, bandwidth, flatness and rolloff: timbre rather than notes. */
    val texture: Double = 1.0,
    /** Onset strength, peak density and local tempo statistics. */
    val rhythm: Double = 0.8,
    /** RMS centre, spread and range: the loudness shape, independent of what is making it. */
    val dynamics: Double = 0.45,
) {

init

    init {
        require(listOf(harmony, pitch, texture, rhythm, dynamics).all { it.isFinite() && it >= 0.0 }) {
            "corpus distance weights must be finite and non-negative"
        }
        require(harmony + pitch + texture + rhythm + dynamics > 0.0) {
            "at least one corpus distance weight must be positive"
        }
    }
}

private fun corpusWindowOperation(
    operation: String,
    source: AssetRegion,
    scalesSeconds: List<Double>,
    overlap: Double,
    hopLength: Int,
    nFft: Int,
    backend: String?,
    requestedBy: String?,
    forceRegenerate: Boolean,
): ModelOperation {
    require(scalesSeconds.isNotEmpty() && scalesSeconds.all { it.isFinite() && it > 0.0 }) {
        "scalesSeconds must contain positive finite values"
    }
    require(overlap >= 0.0 && overlap < 1.0) { "overlap must be in [0, 1)" }
    require(hopLength > 0 && nFft > 0) { "hopLength and nFft must be positive" }
    return ModelOperation(
        operation = operation,
        inputs = listOf(source),
        parameters = buildJsonObject {
            putJsonArray("scales_seconds") { scalesSeconds.forEach { add(JsonPrimitive(it)) } }
            put("overlap", overlap)
            put("hop_length", hopLength)
            put("n_fft", nFft)
        },
        candidateCount = 1,
        backendPreference = backend,
        requestedBy = requestedBy,
        forceRegenerate = forceRegenerate,
    )
}

fun segmentCorpus

Enumerate deterministic source-adjacent windows without computing descriptors.

fun segmentCorpus(
    source: AssetRegion,
    scalesSeconds: List<Double> = listOf(1.5, 3.0, 6.0, 12.0),
    overlap: Double = 0.5,
    backend: String? = "analysis.librosa",
    requestedBy: String? = null,
    forceRegenerate: Boolean = false,
): ModelOperation = corpusWindowOperation(
    "analysis.segment", source, scalesSeconds, overlap, 512, 2048,
    backend, requestedBy, forceRegenerate,
)

fun embedCorpus

Compute and persist grouped descriptors for every multi-scale source window.

fun embedCorpus(
    source: AssetRegion,
    scalesSeconds: List<Double> = listOf(1.5, 3.0, 6.0, 12.0),
    overlap: Double = 0.5,
    hopLength: Int = 512,
    nFft: Int = 2048,
    backend: String? = "analysis.librosa",
    requestedBy: String? = null,
    forceRegenerate: Boolean = false,
): ModelOperation = corpusWindowOperation(
    "analysis.embed", source, scalesSeconds, overlap, hopLength, nFft,
    backend, requestedBy, forceRegenerate,
)

fun findCorpusNeighbours

Retrieve indexed windows by an explicit weighted descriptor distance.

fun findCorpusNeighbours(
    source: AssetRegion,
    scaleSeconds: Double = source.durationSeconds,
    limit: Int = 12,
    weights: CorpusDistanceWeights = CorpusDistanceWeights(),
    includeSameAsset: Boolean = true,
    hopLength: Int = 512,
    nFft: Int = 2048,
    backend: String? = "analysis.librosa",
    requestedBy: String? = null,
    forceRegenerate: Boolean = false,
): ModelOperation {
    require(scaleSeconds > 0.0 && scaleSeconds.isFinite()) { "scaleSeconds must be positive" }
    require(abs(scaleSeconds - source.durationSeconds) < 1e-6) {
        "the query duration must equal scaleSeconds"
    }
    require(limit in 1..64) { "limit must be between 1 and 64" }
    require(hopLength > 0 && nFft > 0) { "hopLength and nFft must be positive" }
    return ModelOperation(
        operation = "analysis.find_neighbours",
        inputs = listOf(source),
        parameters = buildJsonObject {
            put("scale_seconds", scaleSeconds)
            put("limit", limit)
            put("include_same_asset", includeSameAsset)
            put("hop_length", hopLength)
            put("n_fft", nFft)
            putJsonObject("weights") {
                put("harmony", weights.harmony)
                put("pitch", weights.pitch)
                put("texture", weights.texture)
                put("rhythm", weights.rhythm)
                put("dynamics", weights.dynamics)
            }
        },
        candidateCount = 1,
        backendPreference = backend,
        requestedBy = requestedBy,
        forceRegenerate = forceRegenerate,
    )
}