JavaCPP FFMpeg to JavaSound

Question

I have a problem to be able to read audio using JavaCPP FFMpeg library. I don’t know how to pass it to java sound and I don’t know too if my code is correct.

Let’s see the more important part of my code (video is OK so I drop this) :

The variables :

//==========================================================================
// FFMpeg 4.x - Video and Audio
//==========================================================================

private final AVFormatContext   pFormatCtx = new AVFormatContext(null);
private final AVDictionary      OPTIONS_DICT = null;
private AVPacket                pPacket = new AVPacket();
    
//==========================================================================
// FFMpeg 4.x - Audio
//==========================================================================
    
private AVCodec                 pAudioCodec;
private AVCodecContext          pAudioCodecCtx;
private final List<StreamInfo>  audioStreams = new ArrayList<>();
private int                     audio_data_size;
private final BytePointer       audio_data = new BytePointer(0);
private int                     audio_ret;
private AVFrame                 pAudioDecodedFrame = null;
private AVCodecParserContext    pAudioParser;
private SwrContext              audio_swr_ctx = null;

Then I call prepare functions in this order :

private void prepareFirst() throws Exception{
    oldFile = file;
            
    // Initialize packet and check for error
    pPacket = av_packet_alloc();
    if(pPacket == null){
        throw new Exception("ALL: Couldn't allocate packet");
    }

    // Open video file
    if (avformat_open_input(pFormatCtx, file.getPath(), null, null) != 0) {
        throw new Exception("ALL: Couldn't open file");
    }

    // Retrieve stream information
    if (avformat_find_stream_info(pFormatCtx, (PointerPointer)null) < 0) {
        throw new Exception("ALL: Couldn't find stream information");
    }

    // Dump information about file onto standard error
    av_dump_format(pFormatCtx, 0, file.getPath(), 0);

    // Find the first audio/video stream
    for (int i = 0; i < pFormatCtx.nb_streams(); i++) {
        switch(pFormatCtx.streams(i).codecpar().codec_type()){
            case AVMEDIA_TYPE_VIDEO -> videoStreams.add(new StreamInfo(i, pFormatCtx.streams(i)));
            case AVMEDIA_TYPE_AUDIO -> audioStreams.add(new StreamInfo(i, pFormatCtx.streams(i)));
        }
    }
    
    if(videoStreams.isEmpty() && type != PlayType.AudioOnly){
        throw new Exception("Didn't find an audio stream");
    }
    if(audioStreams.isEmpty() && type != PlayType.VideoOnly){
        throw new Exception("Didn't find a video stream");
    }
}

private void prepareAudio() throws Exception{
    //++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
    // AUDIO
    //------------------------------------------------------------------

    if(audioStreams.isEmpty() == false){
        //===========================
        //------------
        
//                // Let's search for AVCodec
//                pAudioCodec = avcodec_find_decoder(pFormatCtx.streams(audioStreams.get(0).getStreamIndex()).codecpar().codec_id());
//                if (pAudioCodec == null) {
//                    throw new Exception("AUDIO: Unsupported codec or not found!");
//                }
//
//                // Let's alloc AVCodecContext
//                pAudioCodecCtx = avcodec_alloc_context3(pAudioCodec);
//                if (pAudioCodecCtx == null) {            
//                    throw new Exception("AUDIO: Unallocated codec context or not found!");
//                }
        
        // Get a pointer to the codec context for the video stream
        pAudioCodecCtx = pFormatCtx.streams(audioStreams.get(0).getStreamIndex()).codec();

        // Find the decoder for the video stream
        pAudioCodec = avcodec_find_decoder(pAudioCodecCtx.codec_id());
        if (pAudioCodec == null) {
            throw new Exception("AUDIO: Unsupported codec or not found!");
        }

        //===========================
        //------------

        /* open it */
        if (avcodec_open2(pAudioCodecCtx, pAudioCodec, OPTIONS_DICT) < 0) {
            throw new Exception("AUDIO: Could not open codec");
        }

        pAudioDecodedFrame = av_frame_alloc();
        if (pAudioDecodedFrame == null){
            throw new Exception("AUDIO: DecodedFrame allocation failed");
        }

        audio_swr_ctx = swr_alloc_set_opts(
                null,                           // existing Swr context or NULL
                AV_CH_LAYOUT_STEREO,            // output channel layout (AV_CH_LAYOUT_*)
                AV_SAMPLE_FMT_S16,              // output sample format (AV_SAMPLE_FMT_*).
                44100,                          // output sample rate (frequency in Hz)
                pAudioCodecCtx.channels(),  // input channel layout (AV_CH_LAYOUT_*)
                pAudioCodecCtx.sample_fmt(),    // input sample format (AV_SAMPLE_FMT_*).
                pAudioCodecCtx.sample_rate(),   // input sample rate (frequency in Hz)
                0,                              // logging level offset
                null                            // parent logging context, can be NULL
        );
        
        swr_init(audio_swr_ctx);
        
        av_samples_fill_arrays(
                pAudioDecodedFrame.data(),      // audio_data,
                pAudioDecodedFrame.linesize(),  // linesize
                audio_data,                     // buf
                (int)AV_CH_LAYOUT_STEREO,       // nb_channels
                44100,                          // nb_samples
                AV_SAMPLE_FMT_S16,              // sample_fmt
                0                               // align
        );
        
    }
    
    // Audio treatment end ---------------------------------------------
    //==================================================================
}

And then when I launch the thread :

private void doPlay() throws Exception{
    av_init_packet(pPacket);

    // Read frames
    while (av_read_frame(pFormatCtx, pPacket) >= 0) {
        if (type != PlayType.AudioOnly && pPacket.stream_index() == videoStreams.get(0).getStreamIndex()) {
            // Is this a packet from the video stream?
            decodeVideo();
            renewPacket();
        }

        if (type != PlayType.VideoOnly && pPacket.stream_index() == audioStreams.get(0).getStreamIndex()) {
            // Is this a packet from the audio stream?
            if(pPacket.size() > 0){
                decodeAudio();
                renewPacket();
            }
        }
    }
}

private void renewPacket(){
    // Free the packet that was allocated by av_read_frame
    av_packet_unref(pPacket);

    pPacket.data(null);
    pPacket.size(0);
    av_init_packet(pPacket);
}

And again, this is where I don’t read audio :

private void decodeAudio() throws Exception{

    do {
        audio_ret = avcodec_send_packet(pAudioCodecCtx, pPacket);
    } while(audio_ret == AVERROR_EAGAIN());
    System.out.println("packet sent return value: " + audio_ret);

    if(audio_ret == AVERROR_EOF || audio_ret == AVERROR_EINVAL()) {
        StringBuilder sb = new StringBuilder();
        Formatter formatter = new Formatter(sb, Locale.US);
        formatter.format("AVERROR(EAGAIN): %d, AVERROR_EOF: %d, AVERROR(EINVAL): %d\n", AVERROR_EAGAIN(), AVERROR_EOF, AVERROR_EINVAL());
        formatter.format("Audio frame getting error (%d)!\n", audio_ret);
        throw new Exception(sb.toString());
    }

    audio_ret = avcodec_receive_frame(pAudioCodecCtx, pAudioDecodedFrame);
    System.out.println("frame received return value: " + audio_ret);

    audio_data_size = av_get_bytes_per_sample(AV_SAMPLE_FMT_S16);

    if (audio_data_size < 0) {
        /* This should not occur, checking just for paranoia */
        throw new Exception("Failed to calculate data size");
    }
    
    double frame_nb = 44100d / pAudioCodecCtx.sample_rate() * pAudioDecodedFrame.nb_samples();
    long out_count = Math.round(Math.floor(frame_nb));

    int out_samples = swr_convert(
            audio_swr_ctx,
            audio_data, 
            (int)out_count,
            pAudioDecodedFrame.data(0),
            pAudioDecodedFrame.nb_samples()
    );
    
    if (out_samples < 0) {
        throw new Exception("AUDIO: Error while converting");
    }
    
    int dst_bufsize = av_samples_get_buffer_size(
        pAudioDecodedFrame.linesize(), 
        (int)AV_CH_LAYOUT_STEREO,  
        out_samples,
        AV_SAMPLE_FMT_S16,
        1
    );
    
    AudioFormat audioFormat = new AudioFormat(
            pAudioDecodedFrame.sample_rate(),
            16,
            2, 
            true, 
            false
    );
    
    BytePointer bytePointer = pAudioDecodedFrame.data(0);
    ByteBuffer byteBuffer = bytePointer.asBuffer();

    byte[] bytes = new byte[byteBuffer.remaining()];
    byteBuffer.get(bytes);
    
    try (SourceDataLine sdl = AudioSystem.getSourceDataLine(audioFormat)) {
        sdl.open(audioFormat);                
        sdl.start();
        sdl.write(bytes, 0, bytes.length);
        sdl.drain();
        sdl.stop();
    } catch (LineUnavailableException ex) {
        Logger.getLogger(AVEntry.class.getName()).log(Level.SEVERE, null, ex);
    }    
}

Do you have an idea ?