Skip to content

AudioDecoder.kt

src/jvmMain/kotlin/afield/graphics/texture/video/AudioDecoder.kt · 406 lines

Decodes an audio stream with FFmpeg into signed 16-bit PCM AudioFrames for AudioPlayer.

package afield.graphics.texture.video

import org.bytedeco.ffmpeg.avcodec.AVCodecContext
import org.bytedeco.ffmpeg.avcodec.AVPacket
import org.bytedeco.ffmpeg.avformat.AVFormatContext
import org.bytedeco.ffmpeg.avutil.AVChannelLayout
import org.bytedeco.ffmpeg.avutil.AVFrame
import org.bytedeco.ffmpeg.global.avcodec.*
import org.bytedeco.ffmpeg.global.avformat.*
import org.bytedeco.ffmpeg.global.avutil.*
import org.bytedeco.ffmpeg.global.swresample.*
import org.bytedeco.ffmpeg.swresample.SwrContext
import org.bytedeco.javacpp.BytePointer
import org.bytedeco.javacpp.PointerPointer
import java.nio.ByteBuffer
import java.nio.ByteOrder

class AudioDecoder(private val audioPath: String)

AudioDecoder wraps FFmpeg functionality to decode audio streams. Decodes audio to PCM format (signed 16-bit) suitable for OpenAL playback.

class AudioDecoder(private val audioPath: String) {
    private var formatContext: AVFormatContext? = null
    private var codecContext: AVCodecContext? = null
    private var swrContext: SwrContext? = null
    private var packet: AVPacket? = null
    private var frame: AVFrame? = null
    private var resampledFrame: AVFrame? = null

var audioStreamIndex: Int = -1

Index of the stream open decodes, or -1 before a successful open. open takes the first stream the container reports as audio, and fails if there is none.

var sampleRate: Int

Sample rate of the source stream in Hz, which need not be the rate of the frames you get: open resamples to the rate it was asked for, and AudioFrame.sampleRate reports that one.

    var sampleRate: Int = 44100
        private set

var channels: Int

Channel count of the source stream, which is likewise the source's and not the output's: open converts to the count it was asked for, and AudioFrame.channels reports that one.

    var channels: Int = 2
        private set

var duration: Double

Length of the audio in seconds, taken from the stream header or, when the stream declares none, from the container's own duration. 0.0 when neither declares one.

    var duration: Double = 0.0
        private set

    private var isOpen = false
    private var targetSampleRate: Int = 44100  // Target sample rate for output
    private var targetChannels: Int = 2  // Target channels for output (stereo)

Buffer for resampled audio data

    private var resampleBuffer: BytePointer? = null
    private var resampleBufferSize: Int = 0

init

    init {
        avformat_network_init()
    }

fun open(outputSampleRate: Int = 44100, outputChannels: Int = 2): Boolean

Open the audio file and initialize decoder. @param outputSampleRate Target sample rate for decoded audio (default 44100 Hz) @param outputChannels Target number of channels (1=mono, 2=stereo, default 2)

body — 119 lines, from line 78
    fun open(outputSampleRate: Int = 44100, outputChannels: Int = 2): Boolean {
        if (isOpen) return true

        this.targetSampleRate = outputSampleRate
        this.targetChannels = outputChannels

        try {
            // Open audio file
            formatContext = avformat_alloc_context()
            if (avformat_open_input(formatContext, audioPath, null, null) != 0) {
                println("Error: Could not open audio file: $audioPath")
                return false
            }

            // Retrieve stream information
            if (avformat_find_stream_info(formatContext, null as PointerPointer<*>?) < 0) {
                println("Error: Could not find stream information")
                return false
            }

            // Find the audio stream
            audioStreamIndex = -1
            for (i in 0 until formatContext!!.nb_streams()) {
                val stream = formatContext!!.streams(i)
                if (stream.codecpar().codec_type() == AVMEDIA_TYPE_AUDIO) {
                    audioStreamIndex = i
                    break
                }
            }

            if (audioStreamIndex == -1) {
                println("Error: Could not find audio stream")
                return false
            }

            val stream = formatContext!!.streams(audioStreamIndex)
            val codecPar = stream.codecpar()

            // Find decoder
            val codec = avcodec_find_decoder(codecPar.codec_id())
            if (codec == null) {
                println("Error: Unsupported audio codec")
                return false
            }

            // Create codec context
            codecContext = avcodec_alloc_context3(codec)
            if (avcodec_parameters_to_context(codecContext, codecPar) < 0) {
                println("Error: Could not copy codec parameters to context")
                return false
            }

            // Open codec
            if (avcodec_open2(codecContext, codec, null as PointerPointer<*>?) < 0) {
                println("Error: Could not open audio codec")
                return false
            }

            // Get audio properties
            sampleRate = codecContext!!.sample_rate()
            channels = codecContext!!.ch_layout().nb_channels()

            // Calculate duration
            val streamDuration = stream.duration()
            val timeBase = stream.time_base()
            if (streamDuration != AV_NOPTS_VALUE) {
                duration = streamDuration * av_q2d(timeBase)
            } else if (formatContext!!.duration() != AV_NOPTS_VALUE) {
                duration = formatContext!!.duration() / AV_TIME_BASE.toDouble()
            }

            println("Audio opened: ${sampleRate}Hz, $channels channels, ${duration}s duration")

            // Initialize resampler if needed
            val needsResampling = sampleRate != targetSampleRate || channels != targetChannels
            if (needsResampling) {
                swrContext = swr_alloc()

                // Configure channel layouts using the new AVChannelLayout API (av_get_default_channel_layout was removed)
                val inChannelLayout = codecContext!!.ch_layout()
                if (inChannelLayout.nb_channels() == 0) {
                    av_channel_layout_default(inChannelLayout, channels)
                }

                val outChannelLayout = AVChannelLayout()
                av_channel_layout_default(outChannelLayout, targetChannels)

                if (av_opt_set_chlayout(swrContext, "in_chlayout", inChannelLayout, 0) < 0 ||
                    av_opt_set_chlayout(swrContext, "out_chlayout", outChannelLayout, 0) < 0) {
                    println("Error: Could not set channel layout on resampler")
                    return false
                }
                av_opt_set_int(swrContext, "in_sample_rate", sampleRate.toLong(), 0)
                av_opt_set_int(swrContext, "out_sample_rate", targetSampleRate.toLong(), 0)
                av_opt_set_sample_fmt(swrContext, "in_sample_fmt", codecContext!!.sample_fmt(), 0)
                av_opt_set_sample_fmt(swrContext, "out_sample_fmt", AV_SAMPLE_FMT_S16, 0)

                if (swr_init(swrContext) < 0) {
                    println("Error: Could not initialize resampler")
                    return false
                }

                println("Audio resampling: ${sampleRate}Hz/$channels ch -> ${targetSampleRate}Hz/$targetChannels ch")
            }

            // Allocate packet and frame
            packet = av_packet_alloc()
            frame = av_frame_alloc()
            resampledFrame = av_frame_alloc()

            isOpen = true
            return true
        } catch (e: Exception) {
            println("Error opening audio: ${e.message}")
            e.printStackTrace()
            close()
            return false
        }
    }

fun decodeFrame(): AudioFrame?

Decode the next audio frame. @return AudioFrame containing PCM data, or null if end of stream or error

body — 115 lines, from line 202
    fun decodeFrame(): AudioFrame? {
        if (!isOpen) return null

        while (true) {
            // Read packet
            val ret = av_read_frame(formatContext, packet)
            if (ret < 0) {
                // End of stream or error
                return null
            }

            // Only process audio stream packets
            if (packet!!.stream_index() != audioStreamIndex) {
                av_packet_unref(packet)
                continue
            }

            // Send packet to decoder
            if (avcodec_send_packet(codecContext, packet) < 0) {
                println("Error sending audio packet to decoder")
                av_packet_unref(packet)
                continue
            }

            av_packet_unref(packet)

            // Receive decoded frame
            val receiveRet = avcodec_receive_frame(codecContext, frame)
            if (receiveRet == AVERROR_EAGAIN() || receiveRet == AVERROR_EOF()) {
                continue
            } else if (receiveRet < 0) {
                println("Error receiving audio frame from decoder")
                return null
            }

            // Calculate timestamp
            val pts = if (frame!!.pts() != AV_NOPTS_VALUE) frame!!.pts() else frame!!.best_effort_timestamp()
            val stream = formatContext!!.streams(audioStreamIndex)
            val timeBase = stream.time_base()
            val timestamp = pts * av_q2d(timeBase)

            // Resample if needed
            val outputData: ByteBuffer
            val outputSamples: Int

            if (swrContext != null) {
                // Calculate output sample count
                val inputSamples = frame!!.nb_samples()
                val maxOutputSamples = av_rescale_rnd(
                    swr_get_delay(swrContext, sampleRate.toLong()) + inputSamples,
                    targetSampleRate.toLong(),
                    sampleRate.toLong(),
                    AV_ROUND_UP
                ).toInt()

                // Allocate buffer if needed
                val requiredSize = maxOutputSamples * targetChannels * 2  // 2 bytes per sample (16-bit)
                if (resampleBuffer == null || resampleBufferSize < requiredSize) {
                    resampleBuffer?.deallocate()
                    resampleBuffer = BytePointer(requiredSize.toLong())
                    resampleBufferSize = requiredSize
                }

                // Resample
                val outputPointers = PointerPointer<BytePointer>(1).put(0, resampleBuffer)
                val inputPointers = frame!!.data()
                val outSamples = swr_convert(
                    swrContext,
                    outputPointers,
                    maxOutputSamples,
                    inputPointers,
                    inputSamples
                )

                if (outSamples < 0) {
                    println("Error resampling audio")
                    return null
                }

                outputSamples = outSamples
                val dataSize = outSamples * targetChannels * 2
                outputData = resampleBuffer!!.asByteBuffer().limit(dataSize).order(ByteOrder.nativeOrder())
            } else {
                // No resampling needed, use frame data directly
                outputSamples = frame!!.nb_samples()

                // Check if data is planar or interleaved
                if (av_sample_fmt_is_planar(codecContext!!.sample_fmt()) != 0) {
                    // Planar format - need to interleave
                    println("Warning: Planar audio format detected - forcing resampling for proper interleaving")
                    // This path shouldn't be reached if swrContext is properly initialized
                    // If it is reached, the audio will be mono-only from first plane
                    // Best practice: always use swrContext for format conversion
                    val dataSize = outputSamples * 2  // Only first channel, 16-bit samples
                    outputData = BytePointer(frame!!.data(0)).asByteBuffer().limit(dataSize).order(ByteOrder.nativeOrder())
                } else {
                    // Already interleaved - verify sample format is S16
                    val sampleFmt = codecContext!!.sample_fmt()
                    if (sampleFmt != AV_SAMPLE_FMT_S16) {
                        println("Warning: Unexpected sample format $sampleFmt, audio may be distorted")
                    }
                    val dataSize = outputSamples * channels * 2  // 2 bytes per sample (16-bit)
                    outputData = BytePointer(frame!!.data(0)).asByteBuffer().limit(dataSize).order(ByteOrder.nativeOrder())
                }
            }

            return AudioFrame(
                data = outputData,
                samples = outputSamples,
                sampleRate = targetSampleRate,
                channels = targetChannels,
                timestamp = timestamp
            )
        }
    }

fun seek(timestamp: Double): Boolean

Seek to a specific timestamp in the audio stream. @param timestamp Time in seconds

    fun seek(timestamp: Double): Boolean {
        if (!isOpen) return false

        val stream = formatContext!!.streams(audioStreamIndex)
        val timeBase = stream.time_base()
        val seekTarget = (timestamp / av_q2d(timeBase)).toLong()

        val ret = av_seek_frame(formatContext, audioStreamIndex, seekTarget, AVSEEK_FLAG_BACKWARD)
        if (ret < 0) {
            println("Error seeking audio to $timestamp")
            return false
        }

        // Flush codec buffers
        avcodec_flush_buffers(codecContext)

        return true
    }

fun close()

Close the decoder and free resources.

    fun close() {
        if (!isOpen) return

        resampleBuffer?.deallocate()
        resampleBuffer = null

        swrContext?.let {
            swr_free(it)
            swrContext = null
        }

        resampledFrame?.let {
            av_frame_free(it)
            resampledFrame = null
        }

        frame?.let {
            av_frame_free(it)
            frame = null
        }

        packet?.let {
            av_packet_free(it)
            packet = null
        }

        codecContext?.let {
            avcodec_free_context(it)
            codecContext = null
        }

        formatContext?.let {
            avformat_close_input(it)
            formatContext = null
        }

        isOpen = false
    }
}

data class AudioFrame

Data class representing a decoded audio frame in PCM format.

data class AudioFrame(
    /**
     * Interleaved PCM samples, signed 16-bit, in native byte order. The decoder owns this buffer
     * and refills it on the next [AudioDecoder.decodeFrame], so queue or copy the samples before
     * decoding again.
     */
    val data: ByteBuffer,
    /** Samples per channel in [data], which therefore holds this many times [channels] values. */
    val samples: Int,
    /**
     * Sample rate of [data] in Hz, which is the rate asked of [AudioDecoder.open] rather than the
     * source's own [AudioDecoder.sampleRate].
     */
    val sampleRate: Int,
    /** Channel count of [data], likewise the count asked of [AudioDecoder.open]: 1 mono, 2 stereo. */
    val channels: Int,
    /** When the first sample is heard, in seconds on the source stream's own timeline. */
    val timestamp: Double
)