OpenCV and Network Cameras -or- How to spy on the neighbors?

Question

---

A bit of context; this program was built originally to work with USB cameras - but because of the setup between where the cameras needs to be and where the computer is it makes more sense to switch to cameras run over a network. Now I'm trying to convert the program to accomplish this, but my efforts thus far have met with poor results. I've also asked this same question over on the OpenCV forums. Help me spy on my neighbors! (This is with their permission, of course!) :D

---

I'm using:

• OpenCV v2.4.6.0
• C++
• D-Link Cloud Camera 7100 (Installer is DCS-7010L, according to the instructions.)

---

I am trying to access the DLink camera's video feed through OpenCV.

I can access the camera through it's IP address with a browser without any issues. Unfourtunately; my program is less cooperative. When attempting to access the camera the program gives the OpenCV-generated error:

warning: Error opening file (../../modules/highgui/src/cap_ffmpeg_impl.hpp:529)

This error occurs with just about everything I try that doesn't somehow generate more problems.

For reference - the code in OpenCV's cap_ffmpeg_impl.hpp around line 529 is as follows:

522    bool CvCapture_FFMPEG::open( const char* _filename )
523    {
524        unsigned i;
525        bool valid = false;
526
527        close();
528
529    #if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0)
530        int err = avformat_open_input(&ic, _filename, NULL, NULL);
531    #else
532        int err = av_open_input_file(&ic, _filename, NULL, 0, NULL);
533    #endif
...
616    }

...for which I have no idea what I'm looking at. It seems to be looking for the ffmpeg version - but I've already installed the latest ffmpeg on that computer, so that shouldn't be the issue.

This is the edited down version I tried to use as per Sebastian Schmitz's recommendation:

 1    #include <fstream>                            // File input/output
 2    #include <iostream>                           // cout / cin / etc
 3    #include <windows.h>                      // Windows API stuff
 4    #include <stdio.h>                            // More input/output stuff
 5    #include <string>                         // "Strings" of characters strung together to form words and stuff
 6    #include <cstring>                            // "Strings" of characters strung together to form words and stuff
 7    #include <streambuf>                      // For buffering load files
 8    #include <array>                          // Functions for working with arrays
 9    #include <opencv2/imgproc/imgproc.hpp>        // Image Processor
10    #include <opencv2/core/core.hpp>          // Basic OpenCV structures (cv::Mat, Scalar)
11    #include <opencv2/highgui/highgui.hpp>        // OpenCV window I/O
12    #include "opencv2/calib3d/calib3d.hpp"
13    #include "opencv2/features2d/features2d.hpp"
14    #include "opencv2/opencv.hpp"
15    #include "resource.h"                     // Included for linking the .rc file
16    #include <conio.h>                            // For sleep()
17    #include <chrono>                         // To get start-time of program.
18    #include <algorithm>                      // For looking at whole sets.
19
20    #ifdef __BORLANDC__
21      #pragma argsused
22    #endif
23
24    using namespace std;                      // Standard operations. Needed for most basic functions.
25    using namespace std::chrono;              // Chrono operations. Needed getting starting time of program.
26    using namespace cv;                           // OpenCV operations. Needed for most OpenCV functions.
27
28    string videoFeedAddress = "";
29    VideoCapture videoFeedIP = NULL;
30    Mat clickPointStorage; //Artifact from original program.
31
32    void displayCameraViewTest()
33    {
34      VideoCapture cv_cap_IP;
35      Mat color_img_IP;
36      int capture;
37      IplImage* color_img;
38      cv_cap_IP.open(videoFeedAddress);
39      Sleep(100);
40      if(!cv_cap_IP.isOpened())
41      {
42          cout << "Video Error: Video input will not work.\n";
43          cvDestroyWindow("Camera View");
44          return;
45      }
46      clickPointStorage.create(color_img_IP.rows, color_img_IP.cols, CV_8UC3);
47      clickPointStorage.setTo(Scalar(0, 0, 0));
48      cvNamedWindow("Camera View", 0); // create window
49      IplImage* IplClickPointStorage = new IplImage(clickPointStorage);
50      IplImage* Ipl_IP_Img;
51      
52      for(;;)
53      {
54          cv_cap_IP.read(color_img_IP);
55          IplClickPointStorage = new IplImage(clickPointStorage);
56          Ipl_IP_Img = new IplImage(color_img_IP);
57          cvAdd(Ipl_IP_Img, IplClickPointStorage, color_img);
58          cvShowImage("Camera View", color_img); // show frame
59          capture = cvWaitKey(10); // wait 10 ms or for key stroke
60          if(capture == 27 || capture == 13 || capture == 32){break;} // if ESC, Return, or space; close window.
61      }
62      cv_cap_IP.release();
63      delete Ipl_IP_Img;
64      delete IplClickPointStorage;
65      cvDestroyWindow("Camera View");
66      return;
67    }
68    
69    int main()
70    {
71      while(1)
72      {
73          cout << "Please Enter Video-Feed Address: ";
74          cin >> videoFeedAddress;
75          if(videoFeedAddress == "exit"){return 0;}
76          cout << "\nvideoFeedAddress: " << videoFeedAddress << endl;
77          displayCameraViewTest();
78          if(cvWaitKey(10) == 27){return 0;}
79      }
80      return 0;
81    }

Using added 'cout's I was able to narrow it down to line 38: "cv_cap_IP.open(videoFeedAddress);"

No value I enter for the videoFeedAddress variable seems to get a different result. I found THIS site that lists a number of possible addresses to connect to it. Since there exists no 7100 anywhere in the list & considering that the install is labeled "DCS-7010L" I used the addresses found next to the DCS-7010L listings. When trying to access the camera most of them can be reached through the browser, confirming that they reach the camera - but they don't seem to affect the outcome when I use them in the videoFeedAddress variable.

I've tried many of them both with and without username:password, the port number (554), and variations on ?.mjpg (the format) at the end.

I searched around and came across a number of different "possible" answers - but none of them seem to work for me. Some of them did give me the idea for including the above username:password, etc stuff, but it doesn't seem to be making a difference. Of course, the number of possible combinations is certainly rather large- so I certainly have not tried all of them (more direction here would be appreciated). Here are some of the links I found:

1. This is one of the first configurations my code was in. No dice.
2. This one is talking about files - not cameras. It also mentions codecs - but I wouldn't be able to watch it in a web browser if that were the problem, right? (Correct me if I'm wrong here...)
3. This one has the wrong error code/points to the wrong line of code!
4. This one mentions compiling OpenCV with ffmpeg support - but I believe 2.4.6.0 already comes with that all set and ready! Otherwise it's not that different from what I've already tried.
5. Now THIS one appears to be very similar to what I have, but the only proposed solution doesn't really help as I had already located a list of connections. I do not believe this is a duplicate, because as per THIS meta discussion I had a lot more information and so didn't feel comfortable taking over someone else's question - especially if I end up needing to add even more information later.

Thank you for reading this far. I realize that I am asking a somewhat specific question - although I would appreciate any advice you can think of regarding OpenCV & network cameras or even related topics.

---

TLDR: Network Camera and OpenCV are not cooperating. I'm unsure if it's the address I'm using to direct the program to the camera or the command I'm using - but no adjustment I make seems to improve the result beyond what I've already done! Now my neighbors will go unwatched!

Answer 1

There's a number of ways to get the video. ffmpeg is not the only way although it's most convenient. To diagnose if ffmpeg is capable of reading the stream, you should use the standalone ffmpeg/ffplay to try to open the url. If it can open directly, it may be some minor things like url formatting such as double slashes(rtsp://IPADDRESS:554/live1.sdp instead of rtsp://IPADDRESS:554//live1.sdp). If it cannot open it directly, it may need some extra commandline switches to make it work. Then you would need to modify opencv's ffmpeg implementation @ line 529 to pass options to avformat_open_input. This may require quite bit of tweaking before you can get a working program.

You can also check if the camera provide a http mjpeg stream by consulting it's manual. I do not have the camera you are using. So I cannot be of much help on this.

Alternatively, I have two suggestions below, which might help you up and running relatively quickly since you mentioned that vlc is working.

method 1

i assume that you can at least open mjpeg url with your existing opencv/ffmpeg combination. since vlc is working, just use vlc to transcode the video into mjpeg like

vlc.exe --ignore-config -I dummy rtsp://admin:admin@10.10.204.111 --sout=#transcode"{vcodec=MJPG,vb=5000,scale=1,acodec=none}:std{access=http,m‌​ux=raw,dst=127.0.0.1:9080/frame.mjpg}"

after that use http://127.0.0.1:9080/frame.mjpg to grab the frame using opencv VideoCapture. this just requires that you have a transcoder program that can convert the incoming stream into mjpeg.

method 2

you can also directly use vlc api programmatically. the following piece of code use vlc to grab the frames. relevant info for compilation

• C:\Program Files (x86)\VideoLAN\VLC\sdk\include
• C:\Program Files (x86)\VideoLAN\VLC\sdk\lib
• libvlc.lib,libvlccore.lib

code

#include "opencv2/highgui/highgui.hpp"
#include <windows.h>
#include <vlc/vlc.h>

using namespace cv;

struct ctx
{
    Mat* image;
    HANDLE mutex;
    uchar* pixels;
};
bool isRunning=true;

Size getsize(const char* path)
{
    libvlc_instance_t *vlcInstance;
    libvlc_media_player_t *mp;
    libvlc_media_t *media;

    const char * const vlc_args[] = {
        "-R",
       "-I", "dummy",
       "--ignore-config",
       "--quiet",

    };
    vlcInstance = libvlc_new(sizeof(vlc_args) / sizeof(vlc_args[0]), vlc_args);
    media = libvlc_media_new_location(vlcInstance, path);
    mp = libvlc_media_player_new_from_media(media);

    libvlc_media_release(media);
    libvlc_video_set_callbacks(mp, NULL, NULL, NULL, NULL);
    libvlc_video_set_format(mp, "RV24",100,100, 100 * 24 / 8); // pitch = width * BitsPerPixel / 8
    libvlc_media_player_play(mp);

    Sleep(2000);//wait a while so that something get rendered so that size info is available
    unsigned int width=640,height=480;
    libvlc_video_get_size(mp,0,&width,&height);


    if(width==0 || height ==0)
    {
        width=640;
        height=480;
    }
    libvlc_media_player_stop(mp);
    libvlc_release(vlcInstance);
    libvlc_media_player_release(mp);
    return Size(width,height);
}


void *lock(void *data, void**p_pixels)
{
    struct ctx *ctx = (struct ctx*)data;
    WaitForSingleObject(ctx->mutex, INFINITE);
    *p_pixels = ctx->pixels;
    return NULL;

}

void display(void *data, void *id){
    (void) data;
    assert(id == NULL);
}

void unlock(void *data, void *id, void *const *p_pixels)
{

    struct ctx *ctx = (struct ctx*)data;
    Mat frame = *ctx->image;
    if(frame.data)
    {
        imshow("frame",frame);
        if(waitKey(1)==27)
        {
            isRunning=false;
            //exit(0);
        }
    }
    ReleaseMutex(ctx->mutex);
}


int main( )
{
    string url="rtsp://admin:admin@10.10.204.111";
    //vlc sdk does not know the video size until it is rendered, so need to play it a bit so that size is     known
    Size sz = getsize(url.c_str());

    // VLC pointers
    libvlc_instance_t *vlcInstance;
    libvlc_media_player_t *mp;
    libvlc_media_t *media;

    const char * const vlc_args[] = {
        "-R",
        "-I", "dummy",
        "--ignore-config", 
        "--quiet", 
    };
    vlcInstance = libvlc_new(sizeof(vlc_args) / sizeof(vlc_args[0]), vlc_args);
    media = libvlc_media_new_location(vlcInstance, url.c_str());
    mp = libvlc_media_player_new_from_media(media);

    libvlc_media_release(media);

    struct ctx* context = ( struct ctx* )malloc( sizeof( *context ) );
    context->mutex = CreateMutex(NULL, FALSE, NULL);
    context->image = new Mat(sz.height, sz.width, CV_8UC3);
    context->pixels = (unsigned char *)context->image->data;

    libvlc_video_set_callbacks(mp, lock, unlock, display, context);
    libvlc_video_set_format(mp, "RV24", sz.width, sz.height, sz.width * 24 / 8); // pitch = width *     BitsPerPixel / 8

    libvlc_media_player_play(mp);
    while(isRunning)
    {
        Sleep(1);
    }

    libvlc_media_player_stop(mp);
    libvlc_release(vlcInstance);
    libvlc_media_player_release(mp);
    free(context);

    return 0;
}