4

I am trying to achieve something straightforward: writing the code that captures a video stream and saves it into an *.mkv file "as-is" (yeah, no demuxing or reencoding or whatever). Just want to store those AVPacket-s and the MKV container looks ready for that.

Note that the question is about ffmpeg library usage, the ffmpeg binary works fine and can be used to save the HLS steam data via the following:
ffmpeg -i https://bitdash-a.akamaihd.net/content/sintel/hls/playlist.m3u8 -c:v copy out.ts
I know that but the goal is to save any (or almost any) stream, thus the MKV. Actually, there is some code that already can save the streams' data, it fails specifically when trying it with HLS.

After some efforts to provide a short but readable MCVE, here's a sample code that reproduces the problem. The focus is on making the output codec work with HLS streams, thus it may lack a lot of things and details, like extra error checks, corner-cases, optimizations, proper timestamp handling, etc.

#include <atomic>
#include <condition_variable>
#include <deque>
#include <functional>
#include <iostream>
#include <memory>
#include <mutex>
#include <thread>

extern "C" {
#include "libavcodec/avcodec.h"
#include "libavfilter/avfilter.h"
#include "libavfilter/buffersink.h"
#include "libavfilter/buffersrc.h"
#include <libavcodec/avcodec.h>
#include <libavdevice/avdevice.h>
#include <libavformat/avformat.h>
#include <libswscale/swscale.h>
}

// Some public stream. The code works with RTSP, RTMP, MJPEG, etc.
// static const char SOURCE_NAME[] = "http://81.83.10.9:8001/mjpg/video.mjpg"; // works!

// My goal was an actual cam streaming via HLS, but here are some random HLS streams
// that reproduce the problem quite well. Playlists may differ, but the error is exactly the same
static const char SOURCE_NAME[] = "http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/sl.m3u8"; // fails!
// static const char SOURCE_NAME[] = "https://bitdash-a.akamaihd.net/content/MI201109210084_1/m3u8s/f08e80da-bf1d-4e3d-8899-f0f6155f6efa.m3u8"; // fails!

using Pkt = std::unique_ptr<AVPacket, void (*)(AVPacket *)>;
std::deque<Pkt> frame_buffer;
std::mutex frame_mtx;
std::condition_variable frame_cv;
std::atomic_bool keep_running{true};

AVCodecParameters *common_codecpar = nullptr;
std::mutex codecpar_mtx;
std::condition_variable codecpar_cv;

void read_frames_from_source(unsigned N)
{
    AVFormatContext *fmt_ctx = avformat_alloc_context();

    int err = avformat_open_input(&fmt_ctx, SOURCE_NAME, nullptr, nullptr);
    if (err < 0) {
        std::cerr << "cannot open input" << std::endl;
        avformat_free_context(fmt_ctx);
        return;
    }

    err = avformat_find_stream_info(fmt_ctx, nullptr);
    if (err < 0) {
        std::cerr << "cannot find stream info" << std::endl;
        avformat_free_context(fmt_ctx);
        return;
    }

    // Simply finding the first video stream, preferrably H.264. Others are ignored below
    int video_stream_id = -1;
    for (unsigned i = 0; i < fmt_ctx->nb_streams; i++) {
        auto *c = fmt_ctx->streams[i]->codecpar;
        if (c->codec_type == AVMEDIA_TYPE_VIDEO) {
            video_stream_id = i;
            if (c->codec_id == AV_CODEC_ID_H264)
                break;
        }
    }

    if (video_stream_id < 0) {
        std::cerr << "failed to find find video stream" << std::endl;
        avformat_free_context(fmt_ctx);
        return;
    }

    {   // Here we have the codec params and can launch the writer
        std::lock_guard<std::mutex> locker(codecpar_mtx);
        common_codecpar = fmt_ctx->streams[video_stream_id]->codecpar;
    }
    codecpar_cv.notify_all();

    unsigned cnt = 0;
    while (++cnt <= N) { // we read some limited number of frames
        Pkt pkt{av_packet_alloc(), [](AVPacket *p) { av_packet_free(&p); }};

        err = av_read_frame(fmt_ctx, pkt.get());
        if (err < 0) {
            std::cerr << "read packet error" << std::endl;
            continue;
        }

        // That's why the cycle above, we write only one video stream here
        if (pkt->stream_index != video_stream_id)
            continue;

        {
            std::lock_guard<std::mutex> locker(frame_mtx);
            frame_buffer.push_back(std::move(pkt));
        }
        frame_cv.notify_one();
    }

    keep_running.store(false);
    avformat_free_context(fmt_ctx);
}

void write_frames_into_file(std::string filepath)
{
    AVFormatContext *out_ctx = nullptr;
    int err = avformat_alloc_output_context2(&out_ctx, nullptr, "matroska", filepath.c_str());
    if (err < 0) {
        std::cerr << "avformat_alloc_output_context2 failed" << std::endl;
        return;
    }

    AVStream *video_stream = avformat_new_stream(out_ctx, avcodec_find_encoder(common_codecpar->codec_id)); // the proper way
    // AVStream *video_stream = avformat_new_stream(out_ctx, avcodec_find_encoder(AV_CODEC_ID_H264)); // forcing the H.264
    // ------>> HERE IS THE TROUBLE, NO CODEC WORKS WITH HLS <<------

    int video_stream_id = video_stream->index;

    err = avcodec_parameters_copy(video_stream->codecpar, common_codecpar);
    if (err < 0) {
        std::cerr << "avcodec_parameters_copy failed" << std::endl;
    }

    if (!(out_ctx->flags & AVFMT_NOFILE)) {
        err =  avio_open(&out_ctx->pb, filepath.c_str(), AVIO_FLAG_WRITE);
        if (err < 0) {
            std::cerr << "avio_open fail" << std::endl;
            return;
        }
    }

    err = avformat_write_header(out_ctx, nullptr); // <<--- ERROR WITH HLS HERE
    if (err < 0) {
        std::cerr << "avformat_write_header failed" << std::endl;
        return; // here we go with hls
    }

    unsigned cnt = 0;
    while (true) {
        std::unique_lock<std::mutex> locker(frame_mtx);
        frame_cv.wait(locker, [&] { return !frame_buffer.empty() || !keep_running; });

        if (!keep_running)
            break;

        Pkt pkt = std::move(frame_buffer.front());
        frame_buffer.pop_front();
        ++cnt;
        locker.unlock();

        pkt->stream_index = video_stream_id; // mandatory
        err = av_write_frame(out_ctx, pkt.get());
        if (err < 0) {
            std::cerr << "av_write_frame failed " << cnt << std::endl;
        } else if (cnt % 25 == 0) {
            std::cout << cnt << " OK" << std::endl;
        }
    }

    av_write_trailer(out_ctx);
    avformat_free_context(out_ctx);
}

int main()
{
    std::thread reader(std::bind(&read_frames_from_source, 1000));
    std::thread writer;

    // Writer wont start until reader's got AVCodecParameters
    // In this example it spares us from setting writer's params properly manually

    {   // Waiting for codec params to be set
        std::unique_lock<std::mutex> locker(codecpar_mtx);
        codecpar_cv.wait(locker, [&] { return common_codecpar != nullptr; });
        writer = std::thread(std::bind(&write_frames_into_file, "out.mkv"));
    }

    reader.join();
    keep_running.store(false);
    writer.join();

    return 0;
}

What happens here? Simply put:

  1. Two threads are spawned, one reads packets from source and stores them in a buffer
  2. The writer waits for the reader to get the AVCodecParameters, so that you can see they are the same being used, almost no manual param setting here
  3. The reader is supposed to read N packets and finish, then the writer follows him. That's how it works with RTSP, RTMP, MJPEG, etc.

What's the problem? Once an HLS stream is tried, there goes the following error:

Tag [27][0][0][0] incompatible with output codec id '27' (H264)

After that the writer segfaults on any write attempt via it's context (that is avformat_write_header here) avformat_write_header fails with an error (see UPD2 below) and thus no successfull write operation is possible.

What's been tried:

  1. Forcing arbitrary codecs (ex.: AV_CODEC_ID_H264). No luck there.
  2. Trying the AV_CODEC_ID_MPEGTS. No way, it's documented as a "fake" codec for internal needs.
  3. Switching some of the multiple options for input or output contexts, no luck there

I'm currenly confused a lot 'coz the error sounds like "Tag H264 is not compatible with codec H264". The ffmpeg logs look like the library managed to comprehend it's dealing with MPEG-TS being sent via HLS, reading is fine but writing into the chosen media container fails:

[hls @ 0x7f94b0000900] Opening 'https://bitdash-a.akamaihd.net/content/MI201109210084_1/video/540_1200000/hls/segment_0.ts' for reading
[hls @ 0x7f94b0000900] Opening 'https://bitdash-a.akamaihd.net/content/MI201109210084_1/video/540_1200000/hls/segment_1.ts' for reading
[hls @ 0x7f94b0000900] Opening 'https://bitdash-a.akamaihd.net/content/MI201109210084_1/video/720_2400000/hls/segment_0.ts' for reading
[hls @ 0x7f94b0000900] Opening 'https://bitdash-a.akamaihd.net/content/MI201109210084_1/video/720_2400000/hls/segment_1.ts' for reading
[hls @ 0x7f94b0000900] Opening 'https://bitdash-a.akamaihd.net/content/MI201109210084_1/video/1080_4800000/hls/segment_0.ts' for reading
[hls @ 0x7f94b0000900] Opening 'https://bitdash-a.akamaihd.net/content/MI201109210084_1/video/1080_4800000/hls/segment_1.ts' for reading
[hls @ 0x7f94b0000900] Could not find codec parameters for stream 0 (Audio: aac ([15][0][0][0] / 0x000F), 0 channels, 112 kb/s): unspecified sample rate
Consider increasing the value for the 'analyzeduration' and 'probesize' options
[matroska @ 0x7f94a8000900] Tag [27][0][0][0] incompatible with output codec id '27' (H264)
avformat_write_header failed
Segmentation fault (core dumped)

No hard googling helped, I'm a bit desperate.
Plz, share your ideas, would be grateful for any.

UPD

  • ffmpeg -i https://bitdash-a.akamaihd.net/content/sintel/hls/playlist.m3u8 out.mkv works fine
  • ffmpeg -i http://qthttp.apple.com.edgesuite.net/1010qwoeiuryfg/sl.m3u8 -c:v copy out.mkv also works fine

... which means ffmpeg can do the trick and the desired result can be achieved

UPD2

It occured that the tag error can be suppressed via
out_ctx->strict_std_compliance = FF_COMPLIANCE_UNOFFICIAL;
I assume it's smth about spelling the "h264" properly in a string tag, doesn't look serious.

Also, after a closer look it occured that it's av_write_frame that actually segfaults. No wonder -- with HLS streams avformat_write_header fails and returns error:

Invalid data found when processing input

That still leaves me with no clues, where's the problem here =((

MasterAler
  • 1,614
  • 3
  • 23
  • 35

1 Answers1

1

Okaaay... after significant efforts in debugging and searching for the answer, it looks like there is a recipe and it's not that complex.
I'll leave it here, so that if anyone else stumbles upon the same magic, he would not get sidetracked.

First of all this question already contains a critical detail that one should know when trying remuxing into MKV. The answer from FFMPEG's maintainer is quite accurate there.

But...

  1. AVCodecContext is somehow mandatory. Maybe that's obvious to everyone, but it was not for me. Looked quite natural to copy the input stream's codecpar directly into the output stream's codecpar. Okay, probably not some blind copying, the ffmpeg docs warn against that, but still these are AVCodecParameters, why not? Alas, the code never worked properly without opening a codec context.
  2. AV_CODEC_FLAG_GLOBAL_HEADER is the key to the solution for sure. There's a mention in AVOutputFormat::flags about AVFMT_GLOBALHEADER, but the exact way to use it (can be found in ffmpeg sources and examples) is as shown in the code snippet below
  3. FF_COMPLIANCE_UNOFFICIAL appeared to be mandatory as well for a fair amount of hls streams (at least those being at hand), otherwise ffmpeg thinks that the code tries to remux packets between different codecs (yeah, because of codec name spelling), which is a slightly different story. Assume it the difference between using the ffmpeg tool with -c:v copy specified and without it.

Here's the necessary update to my code that makes everything working as expected:

void write_frames_into_file(std::string filepath)
{
    AVFormatContext *out_ctx = nullptr;

    int err = avformat_alloc_output_context2(&out_ctx, nullptr, "matroska", filepath.c_str());
    if (err < 0) {
        std::cerr << "avformat_alloc_output_context2 failed" << std::endl;
        return;
    }
    out_ctx->strict_std_compliance = FF_COMPLIANCE_UNOFFICIAL; // !!! (3)

    AVCodec* codec = avcodec_find_encoder(common_codecpar->codec_id);
    AVStream *video_stream = avformat_new_stream(out_ctx, codec); // the proper way

    int video_stream_id = video_stream->index;

    AVCodecContext *encoder = avcodec_alloc_context3(codec);
    avcodec_parameters_to_context(encoder, common_codecpar);
    encoder->time_base = time_base;
    encoder->framerate = frame_rate;
    if (out_ctx->oformat->flags & AVFMT_GLOBALHEADER) // !!! (2)
        encoder->flags |= AV_CODEC_FLAG_GLOBAL_HEADER;

    err = avcodec_open2(encoder, codec, nullptr); // !!! (1)
    if (err < 0) {
        std::cerr << "avcodec_open2 failed" << std::endl;
        return;
    }

    err = avcodec_parameters_from_context(video_stream->codecpar, encoder);
    if (err < 0) {
        std::cerr << "avcodec_parameters_from_context failed" << std::endl;
        return;
    }

    if (!(out_ctx->flags & AVFMT_NOFILE)) {
        err =  avio_open(&out_ctx->pb, filepath.c_str(), AVIO_FLAG_WRITE);
        if (err < 0) {
            std::cerr << "avio_open fail" << std::endl;
            return;
        }
    }

    err = avformat_write_header(out_ctx, nullptr);
    if (err < 0) {
        char ffmpeg_err_buf[AV_ERROR_MAX_STRING_SIZE];
        av_make_error_string(&ffmpeg_err_buf[0], AV_ERROR_MAX_STRING_SIZE, err);
        std::cerr << "avformat_write_header failed: " << ffmpeg_err_buf << std::endl;
        return;
    }
    
    // ....
    // Writing AVPackets here, as in the question, or the other way you wanted to do it
    // ....
}
MasterAler
  • 1,614
  • 3
  • 23
  • 35