How to overlay a picture over a video stream with Gstreamer in C?

Question

I want to overlay a ".png" picture over a stream coming from an IP Camera using Gstreamer. A working pipeline for my hardware is:

gst-launch-1.0 
rtspsrc location=rtsp://user:pass@IP:port/channel latency=400 ! rtph264depay ! 
vpudec use-vpu-memory=false ! imxvideoconvert_ipu 
! video/x-raw,format=I420 ! gdkpixbufoverlay 
location=/home/user/folder/image.png offset-x=100 offset-y=100 ! overlaysink

The problem comes when I try to translate this pipeline in C. The code I wrote for this pipeline runs, but there is no video playback on the display. The player stuck itself before setting the pipeline on "playing" state. Here, there is a simple version of my C implementation:

#include <gst/gst.h>
#include <glib.h>
#include <iostream>

typedef struct _CustomData {
  GstElement *source;
  GstElement *rtp;
  GstElement *sink;
  GstElement *vpudec;
  GstElement *converter, *gdkpixbufoverlay, *capsfilter ;
  GstBus *bus;
  GstElement    *pipeline;
  GMainLoop     *loop;
} CustomData;


static gboolean bus_call (GstBus *bus,
      GstMessage *msg,
      gpointer    data)
{
  GMainLoop *loop = (GMainLoop *) data;

  switch (GST_MESSAGE_TYPE (msg)) {
case GST_MESSAGE_EOS:{
  g_print ("End of stream\n");
  g_main_loop_quit (loop);
  break;
}  
case GST_MESSAGE_ERROR: {
  gchar  *debug;
  GError *error;
  gst_message_parse_error (msg, &error, &debug);
  g_free (debug);
  g_printerr ("Error: %s\n", error->message);
  g_error_free (error);    
  g_main_loop_quit (loop);
  break;
}
default:
  break;
  }    
  return TRUE;
}

static void pad_added_handler (GstElement *src, GstPad *new_pad, CustomData *data) {
  GstPad *sink_pad = gst_element_get_static_pad (data->rtp, "sink");
  GstPadLinkReturn ret;
  GstCaps *new_pad_caps = NULL;
  GstStructure *new_pad_struct = NULL;
  const gchar *new_pad_type = NULL;

  if (gst_pad_is_linked (sink_pad)) {
  goto exit;
  }

  new_pad_caps = gst_pad_query_caps (new_pad, NULL);
  new_pad_struct = gst_caps_get_structure (new_pad_caps, 0);
  new_pad_type = gst_structure_get_name (new_pad_struct);
  if (!g_str_has_prefix (new_pad_type, "application/x-rtp")) {
  g_print ("  It has type '%s' which is not x-rtp . Ignoring.\n", 
  new_pad_type);
  goto exit;
  }

  ret = gst_pad_link (new_pad, sink_pad);
  if (GST_PAD_LINK_FAILED (ret)) {
  g_print("  Type is '%s' but link failed.\n", new_pad_type);
  } else {
  g_print ("  Link succeeded (type '%s').\n", new_pad_type);
  }

exit:

  if (new_pad_caps != NULL)
  gst_caps_unref (new_pad_caps);
  gst_object_unref (sink_pad);
}



int main (int   argc, char *argv[]){

CustomData data;

gst_init (NULL, NULL);

data.loop = g_main_loop_new (NULL, FALSE);

// Create gstreamer elements
data.pipeline = gst_pipeline_new ("player");

data.source           = gst_element_factory_make ("rtspsrc",  "source");
data.rtp              = gst_element_factory_make ("rtph264depay","rtp");
data.vpudec           = gst_element_factory_make ("vpudec","vpudec");
data.converter        = gst_element_factory_make 
("imxcompositor_ipu","converter");
data.capsfilter       = gst_element_factory_make ("capsfilter", "video-
rate");
data.gdkpixbufoverlay = gst_element_factory_make 
("gdkpixbufoverlay","overlaytool");
data.sink             = gst_element_factory_make ("overlaysink", 
"videoSink");


if (!data.pipeline || !data.source || !data.rtp || !data.vpudec || 
!data.converter || !data.capsfilter || !data.gdkpixbufoverlay || !data.sink) 
{
 g_printerr ("One element could not be created. Exiting.\n");
 return -1;
}

g_object_set (data.source, "location","rtsp://user:pass@IP:port/channel", 
NULL);
g_object_set (data.source,"latency", 400 , NULL);
g_object_set (data.vpudec, "use-vpu-memory", false, NULL);

g_object_set (data.gdkpixbufoverlay, 
"location","/home/user/folder/image.png", NULL);
g_object_set (data.gdkpixbufoverlay, "offset-x", 100 , NULL);
g_object_set (data.gdkpixbufoverlay, "offset-y", 100 , NULL);


GstCaps *capsFormat = gst_caps_from_string ("video/x-raw,format=I420");
g_object_set ( data.capsfilter, "caps", capsFormat, NULL);
gst_caps_unref(capsFormat);

//add all elements into the pipeline
gst_bin_add_many (GST_BIN (data.pipeline),
          data.source,
          data.rtp,
          data.vpudec,
          data.converter,
          data.capsfilter,
          data.gdkpixbufoverlay,
          data.sink,
          NULL);

// link all elements
gst_element_link_many ( data.rtp, data.vpudec , data.converter , 
data.capsfilter, data.gdkpixbufoverlay, data.sink, NULL);

g_signal_connect (data.source, "pad-added", G_CALLBACK (pad_added_handler), 
&data);

// Set the pipeline to "playing" state
GstStateChangeReturn ret;
ret = gst_element_set_state (data.pipeline, GST_STATE_PLAYING);

if (ret == GST_STATE_CHANGE_FAILURE) {
   g_printerr("Unable to set the pipeline to the playing state.\n");
   gst_object_unref (data.pipeline);
   return -1;
}


// Iterate
g_main_loop_run (data.loop);

 // Out of the main loop, clean
 g_print ("Returned, stopping playback\n");
 gst_element_set_state (data.pipeline, GST_STATE_NULL);

 g_print ("Deleting pipeline\n");
 gst_object_unref (GST_OBJECT (data.pipeline));

 return 0;
}

Does anyone see the problem?

Thank you

Do you know about `gst_parse_launch()`? It exists for easy construction of pipelines. Basically you can use your gst-launch-1.0 syntax for creating the same pipeline. — Florian Zwoch, Jan 16 '18 at 20:55
Thank you for the suggestion. It works, but in future I will need to have more control and maybe this can be just a temporary solution. It remains the weird fact that the same pipeline in C causes problems, unless there are errors. Have you maybe seen something wrong or not clear? — Pelux, Jan 17 '18 at 09:37
What do you mean more control? Once the pipeline is build you can access everything as you would do the hard way. Check `gst_bin_find_by_name()` to access elements or `gst_bin_iterate_elements()`. From there you can access pads, probes.. well everything. IT just helps you to avoid the hidden mistakes you experience right now. If you want it the hard way do error checking after each step.. — Florian Zwoch, Jan 17 '18 at 10:04
For control I mean exactly the access to the pads. I am pretty new in Gstreamer and I didn't know about this possibility, I thought that gst_parse_launch () was only to test a pipeline without making further actions. Thank you again. — Pelux, Jan 17 '18 at 13:24

score 0 · Answer 1 · answered Jan 24 '18 at 16:29

0

After many tries, I did figure out that in the C code I posted I have selected the wrong element, therefore the data.converter element is:

data.converter = gst_element_factory_make("imxvideoconvert_ipu ","converter");

and not imxcompositor_ipu .

answered Jan 24 '18 at 16:29

Pelux

127
1
4

How to overlay a picture over a video stream with Gstreamer in C?

1 Answers1