19

I'm trying to scan 2 images (32bppArgb format), identify when there is a difference and store the difference block's bounds in a list of rectangles.

Suppose these are the images: enter image description here

second: enter image description here

I want to get the different rectangle bounds (the opened directory window in our case).

This is what I've done:

private unsafe List<Rectangle> CodeImage(Bitmap bmp, Bitmap bmp2)
{

    List<Rectangle> rec = new List<Rectangle>();
    bmData = bmp.LockBits(new System.Drawing.Rectangle(0, 0, 1920, 1080), System.Drawing.Imaging.ImageLockMode.ReadOnly, bmp.PixelFormat);
    bmData2 = bmp2.LockBits(new System.Drawing.Rectangle(0, 0, 1920, 1080), System.Drawing.Imaging.ImageLockMode.ReadOnly, bmp2.PixelFormat);

    IntPtr scan0 = bmData.Scan0;
    IntPtr scan02 = bmData2.Scan0;
    int stride = bmData.Stride;
    int stride2 = bmData2.Stride;
    int nWidth = bmp.Width;
    int nHeight = bmp.Height;
    int minX = int.MaxValue;;
    int minY = int.MaxValue;
    int maxX = 0;
    bool found = false;

    for (int y = 0; y < nHeight; y++) 
    {
        byte* p = (byte*)scan0.ToPointer();
        p += y * stride;
        byte* p2 = (byte*)scan02.ToPointer();
        p2 += y * stride2;
        for (int x = 0; x < nWidth; x++) 
        {

            if (p[0] != p2[0] || p[1] != p2[1] || p[2] != p2[2] || p[3] != p2[3]) //found differences-began to store positions.
            {
                found = true;
                if (x < minX)
                    minX = x;
                if (x > maxX)
                    maxX = x;
                if (y < minY)
                    minY = y;

            } 
            else 
            {

                if (found) 
                {

                    int height = getBlockHeight(stride, scan0, maxX, minY, scan02, stride2);
                    found = false;
                    Rectangle temp = new Rectangle(minX, minY, maxX - minX, height);
                    rec.Add(temp);
                    //x += minX;
                    y += height;
                    minX = int.MaxValue;
                    minY = int.MaxValue;
                    maxX = 0;
                }
            } 
            p += 4;
            p2 += 4;
        }
    }

    return rec;
}

public unsafe int getBlockHeight(int stride, IntPtr scan, int x, int y1, IntPtr scan02, int stride2) //a function to get  an existing block height.
{
    int height = 0;;
    for (int y = y1; y < 1080; y++) //only for example- in our case its 1080 height.
    {
        byte* p = (byte*)scan.ToPointer();
        p += (y * stride) + (x * 4); //set the pointer to a specific potential point. 
        byte* p2 = (byte*)scan02.ToPointer();
        p2 += (y * stride2) + (x * 4); //set the pointer to a specific potential point. 
        if (p[0] != p2[0] || p[1] != p2[1] || p[2] != p2[2] || p[3] != p2[3]) //still change on the height in the increasing **y** of the block.
            height++;
    }

    return height;
}

This is actually how I call the method:

Bitmap a = Image.FromFile(@"C:\Users\itapi\Desktop\1.png") as Bitmap;//generates a 32bppRgba bitmap;
Bitmap b = Image.FromFile(@"C:\Users\itapi\Desktop\2.png") as Bitmap;//

List<Rectangle> l1 = CodeImage(a, b);
int i = 0;
foreach (Rectangle rec in l1)
{
    i++;
    Bitmap tmp = b.Clone(rec, a.PixelFormat);
    tmp.Save(i.ToString() + ".png");
}

But I'm not getting the exact rectangle.. I'm getting only half of that and sometimes even worse. I think something in the code's logic is wrong.

Code for @nico

private unsafe List<Rectangle> CodeImage(Bitmap bmp, Bitmap bmp2) 
{
    List<Rectangle> rec = new List<Rectangle>();
    var bmData1 = bmp.LockBits(new System.Drawing.Rectangle(0, 0, bmp.Width, bmp.Height), System.Drawing.Imaging.ImageLockMode.ReadOnly, bmp.PixelFormat);
    
    var bmData2 = bmp2.LockBits(new System.Drawing.Rectangle(0, 0, bmp.Width, bmp.Height), System.Drawing.Imaging.ImageLockMode.ReadOnly, bmp2.PixelFormat);

    int bytesPerPixel = 3;

    IntPtr scan01 = bmData1.Scan0;
    IntPtr scan02 = bmData2.Scan0;
    int stride1 = bmData1.Stride;
    int stride2 = bmData2.Stride;
    int nWidth = bmp.Width;
    int nHeight = bmp.Height;

    bool[] visited = new bool[nWidth * nHeight];

    byte* base1 = (byte*)scan01.ToPointer();
    byte* base2 = (byte*)scan02.ToPointer();

    for (int y = 0; y < nHeight; y += 5) 
    {
        byte* p1 = base1;
        byte* p2 = base2;

        for (int x = 0; x < nWidth; x += 5) 
        {
            if (!ArePixelsEqual(p1, p2, bytesPerPixel) && !(visited[x + nWidth * y])) 
            {
                // fill the different area
                int minX = x;
                int maxX = x;
                int minY = y;
                int maxY = y;

                var pt = new Point(x, y);

                Stack<Point> toBeProcessed = new Stack<Point> ();
                visited[x + nWidth * y] = true;
                toBeProcessed.Push(pt);
                
                while (toBeProcessed.Count > 0) 
                {
                    var process = toBeProcessed.Pop();
                    var ptr1 = (byte*)scan01.ToPointer() + process.Y * stride1 + process.X * bytesPerPixel;
                    var ptr2 = (byte*) scan02.ToPointer() + process.Y * stride2 + process.X * bytesPerPixel;
                    //Check pixel equality
                    if (ArePixelsEqual(ptr1, ptr2, bytesPerPixel))
                        continue;

                    //This pixel is different
                    //Update the rectangle
                    if (process.X < minX) minX = process.X;
                    if (process.X > maxX) maxX = process.X;
                    if (process.Y < minY) minY = process.Y;
                    if (process.Y > maxY) maxY = process.Y;

                    Point n;
                    int idx;
                    
                    //Put neighbors in stack
                    if (process.X - 1 >= 0) 
                    {
                        n = new Point(process.X - 1, process.Y);
                        idx = n.X + nWidth * n.Y;
                        if (!visited[idx]) 
                        {
                            visited[idx] = true;
                            toBeProcessed.Push(n);
                        }
                    }

                    if (process.X + 1 < nWidth) 
                    {
                        n = new Point(process.X + 1, process.Y);
                        idx = n.X + nWidth * n.Y;
                        if (!visited[idx]) 
                        {
                            visited[idx] = true;
                            toBeProcessed.Push(n);
                        }
                    }

                    if (process.Y - 1 >= 0) 
                    {
                        n = new Point(process.X, process.Y - 1);
                        idx = n.X + nWidth * n.Y;
                        if (!visited[idx]) 
                        {
                            visited[idx] = true;
                            toBeProcessed.Push(n);
                        }
                    }

                    if (process.Y + 1 < nHeight) 
                    {
                        n = new Point(process.X, process.Y + 1);
                        idx = n.X + nWidth * n.Y;
                        if (!visited[idx]) 
                        {
                            visited[idx] = true;
                            toBeProcessed.Push(n);
                        }
                    }
                }

                if (((maxX - minX + 1) > 5) & ((maxY - minY + 1) > 5))
                    rec.Add(new Rectangle(minX, minY, maxX - minX + 1, maxY - minY + 1));
            }

            p1 += 5 * bytesPerPixel;
            p2 += 5 * bytesPerPixel;
        }

        base1 += 5 * stride1;
        base2 += 5 * stride2;
    }

    bmp.UnlockBits(bmData1);
    bmp2.UnlockBits(bmData2);

    return rec;
    
}
spaleet
  • 838
  • 2
  • 10
  • 23
Slashy
  • 1,841
  • 3
  • 23
  • 42
  • When you find your first pixel mismatch, you don't enter your `if (found)` block (because it's in the `else` branch of your detection). Wouldn't you want that block to execute immediately on the pixel on which you found was mismatched? (If you're still having issues working out the logic, consider first implementing it without the whole stride/scan/pixel-bytes and using [`GetPixel`](https://msdn.microsoft.com/en-us/library/system.drawing.bitmap.getpixel%28v=vs.110%29.aspx) instead. Get a _simplist as possible_ solution that works, then optimize it for performance after.) – Chris Sinclair Aug 16 '15 at 13:18
  • In addition what Chris said, check follow it through in a debugger. See what happens – D. Ben Knoble Aug 16 '15 at 13:19
  • @ChrisSinclair allright.. but why it doesnt enter the `if` block? i think it does enter. because on the first mismatch i set `found` to `True`. – Slashy Aug 16 '15 at 13:23
  • @Slashy: Because it's in the `else` block of your detection. When you find your first match, you wait until your next iteration to enter it. Furthermore (on second look) you _only_ enter it _if_ you've detected a mismatch on a _previous_ iteration **and** the current pixel is a mismatch. If the pixel happens to be the same colour, you don't enter it. Check your `if/else` branching logic and ensure this is correct for your purposes. – Chris Sinclair Aug 16 '15 at 14:10
  • @ChrisSinclair oh yes, i know that. that's because i need to find the width of the detected block.. i'll try somthing a come back to you. – Slashy Aug 16 '15 at 14:15
  • @ChrisSinclair no.. i just cant get this working.. breaking my head :( – Slashy Aug 16 '15 at 14:56
  • @Slashy: Simplify the steps and remove complexity. Refactor logical blocks into separate methods so your high level algorithm reads like pseudocode. (don't worry about performance) Make super-simple small test images (say a 6x6 white, and a second 6x6 white with a 3x3 red block in it) to debug with. Start stepping through with the debugger validating the conditional checks and refactored methods for correctness. If you still can't find the issue, do the same work by hand on a sheet of paper as the debugger steps through the lines. If everything still matches, then maybe your algorithm is off. – Chris Sinclair Aug 16 '15 at 15:08
  • Still looking for answers. @ChrisSinclair i'll try . – Slashy Aug 16 '15 at 19:37

4 Answers4

7

I see a couple of problems with your code. If I understand it correctly, you

  1. find a pixel that's different between the two images.
  2. then you continue to scan from there to the right, until you find a position where both images are identical again.
  3. then you scan from the last "different" pixel to the bottom, until you find a position where both images are identical again.
  4. then you store that rectangle and start at the next line below it

enter image description here

Am I right so far?

Two obvious things can go wrong here:

  • If two rectangles have overlapping y-ranges, you're in trouble: You'll find the first rectangle fine, then skip to the bottom Y-coordinate, ignoring all the pixels left or right of the rectangle you just found.
  • Even if there is only one rectangle, you assume that every pixel on the rectangle's border is different, and all the other pixels are identical. If that assumption isn't valid, you'll stop searching too early, and only find parts of rectangles.

If your images come from a scanner or digital camera, or if they contain lossy compression (jpeg) artifacts, the second assumption will almost certainly be wrong. To illustrate this, here's what I get when I mark every identical pixel the two jpg images you linked black, and every different pixel white:

enter image description here

What you see is not a rectangle. Instead, a lot of pixels around the rectangles you're looking for are different:

enter image description here

That's because of jpeg compression artifacts. But even if you used lossless source images, pixels at the borders might not form perfect rectangles, because of antialiasing or because the background just happens to have a similar color in that region.

You could try to improve your algorithm, but if you look at that border, you will find all kinds of ugly counterexamples to any geometric assumptions you'll make.

It would probably be better to implement this "the right way". Meaning:

  • Either implement a flood fill algorithm that erases different pixels (e.g. by setting them to identical or by storing a flag in a separate mask), then recursively checks if the 4 neighbor pixels.
  • Or implement a connected component labeling algorithm, that marks each different pixel with a temporary integer label, using clever data structures to keep track which temporary labels are connected. If you're only interested in a bounding box, you don't even have to merge the temporary labels, just merge the bounding boxes of adjacent labeled areas.

Connected component labeling is in general a bit faster, but is a bit trickier to get right than flood fill.

One last advice: I would rethink your "no 3rd party libraries" policy if I were you. Even if your final product will contain no 3rd party libraries, development might by a lot faster if you used well-documented, well-tested, useful building blocks from a library, then replaced them one by one with your own code. (And who knows, you might even find an open source library with a suitable license that's so much faster than your own code that you'll stick with it in the end...)


ADD: In case you want to rethink your "no libraries" position: Here's a quick and simple implementation using AForge (which has a more permissive library than emgucv):

private static void ProcessImages()
{
    (* load images *)
    var img1 = AForge.Imaging.Image.FromFile(@"compare1.jpg");
    var img2 = AForge.Imaging.Image.FromFile(@"compare2.jpg");

    (* calculate absolute difference *)
    var difference = new AForge.Imaging.Filters.ThresholdedDifference(15)
        {OverlayImage = img1}
        .Apply(img2);

    (* create and initialize the blob counter *)
    var bc = new AForge.Imaging.BlobCounter();
    bc.FilterBlobs = true;
    bc.MinWidth = 5;
    bc.MinHeight = 5;

    (* find blobs *)
    bc.ProcessImage(difference);

    (* draw result *)
    BitmapData data = img2.LockBits(
       new Rectangle(0, 0, img2.Width, img2.Height),
          ImageLockMode.ReadWrite, img2.PixelFormat);

    foreach (var rc in bc.GetObjectsRectangles())
        AForge.Imaging.Drawing.FillRectangle(data, rc, Color.FromArgb(128,Color.Red));

    img2.UnlockBits(data);
    img2.Save(@"compareResult.jpg");
}

The actual difference + blob detection part (without loading and result display) takes about 43ms, for the second run (this first time takes longer of course, due to JITting, cache, etc.)

Result (the rectangle is larger due to jpeg artifacts):

enter image description here

Niki
  • 15,662
  • 5
  • 48
  • 74
  • I agree about the first 2 points but actually it's a png image. ..why I can't get the the edges? – Slashy Aug 18 '15 at 18:50
  • even if i wanted to use 3rd party libraries, such as emgucv- i haven't seen so far somthing which could match my needs... – Slashy Aug 20 '15 at 16:07
  • @Slashy: Honestly, I don't know much about C# image processing libraries, but I'm relatively sure that most of them would have functions to calculate the pixel-wise absolute difference between two images, binarize the result and find connected components (and their bounding boxes) in it. – Niki Aug 20 '15 at 16:24
  • awesome man! but would you agree man that applying diff and only then counting blobs is actually... how to say it-needless. couldnt we found the different blob directly? i know that im asking too much.. but im breaking my head on this searched so many algorithms from dividing image into blocks-to scan with unsafe pointers so haha yeah.. i would like to acheive somthing good :) i appreciate your answers so so so so so much! – Slashy Aug 20 '15 at 18:02
  • @Slashy: Small update: I just realized there's a `ThresholdedDifference` filter that returns an 8-bit image. Using this makes the whole thing run about twice as fast. – Niki Aug 20 '15 at 18:27
  • Regarding your question: You'll have to calculate the difference, either way (what do you think `==` does?). But yes, if you used a single-pass algorithm, you could potentially save one memory write/memory read operation per pixel. AForge is open source, you could try to modify the algorithm so it uses a two-picture comparison instead of each pixel access. Can't say how much time that would save. – Niki Aug 20 '15 at 18:32
  • Also, if you're not too picky about library licenses, I would rather try different libraries, looking for one that can use you CPU's SIMD instructions. That should be *much* faster, even if you need two passes over the data. For example, Intel's IPP can do all these things, but it's not as easy to use as e.g. AForge (in C#, at least). – Niki Aug 20 '15 at 18:39
  • seems that's fast enough.. haha . i'll just try to use the aforge library now. one last thing- if i want to someone also to use that program- he doesnt have to install Aforge too.. right? i just can include the dll's files. Am i right? – Slashy Aug 20 '15 at 18:45
  • Let us [continue this discussion in chat](http://chat.stackoverflow.com/rooms/87519/discussion-between-nikie-and-slashy). – Niki Aug 20 '15 at 18:50
2

Here is a flood-fill based version of your code. It checks every pixel for difference. If it finds a different pixel, it runs an exploration to find the entire different area.

The code is only meant as an illustration. There are certainly some points that could be improved.

unsafe bool ArePixelsEqual(byte* p1, byte* p2, int bytesPerPixel)
{
    for (int i = 0; i < bytesPerPixel; ++i)
        if (p1[i] != p2[i])
            return false;
    return true;
}

private static unsafe List<Rectangle> CodeImage(Bitmap bmp, Bitmap bmp2)
{
    if (bmp.PixelFormat != bmp2.PixelFormat || bmp.Width != bmp2.Width || bmp.Height != bmp2.Height)
        throw new ArgumentException();

    List<Rectangle> rec = new List<Rectangle>();
    var bmData1 = bmp.LockBits(new System.Drawing.Rectangle(0, 0, bmp.Width, bmp.Height), System.Drawing.Imaging.ImageLockMode.ReadOnly, bmp.PixelFormat);
    var bmData2 = bmp2.LockBits(new System.Drawing.Rectangle(0, 0, bmp.Width, bmp.Height), System.Drawing.Imaging.ImageLockMode.ReadOnly, bmp2.PixelFormat);

    int bytesPerPixel = Image.GetPixelFormatSize(bmp.PixelFormat) / 8;        

    IntPtr scan01 = bmData1.Scan0;
    IntPtr scan02 = bmData2.Scan0;
    int stride1 = bmData1.Stride;
    int stride2 = bmData2.Stride;
    int nWidth = bmp.Width;
    int nHeight = bmp.Height;

    bool[] visited = new bool[nWidth * nHeight];

    byte* base1 = (byte*)scan01.ToPointer();
    byte* base2 = (byte*)scan02.ToPointer();

        for (int y = 0; y < nHeight; y++)
        {
            byte* p1 = base1;
            byte* p2 = base2;

            for (int x = 0; x < nWidth; ++x)
            {
                if (!ArePixelsEqual(p1, p2, bytesPerPixel) && !(visited[x + nWidth * y]))
                {
                    // fill the different area
                    int minX = x;
                    int maxX = x;
                    int minY = y;
                    int maxY = y;

                    var pt = new Point(x, y);

                    Stack<Point> toBeProcessed = new Stack<Point>();
                    visited[x + nWidth * y] = true;
                    toBeProcessed.Push(pt);
                    while (toBeProcessed.Count > 0)
                    {
                        var process = toBeProcessed.Pop();
                        var ptr1 = (byte*)scan01.ToPointer() + process.Y * stride1 + process.X * bytesPerPixel;
                        var ptr2 = (byte*)scan02.ToPointer() + process.Y * stride2 + process.X * bytesPerPixel;
                        //Check pixel equality
                        if (ArePixelsEqual(ptr1, ptr2, bytesPerPixel))
                            continue;

                        //This pixel is different
                        //Update the rectangle
                        if (process.X < minX) minX = process.X;
                        if (process.X > maxX) maxX = process.X;
                        if (process.Y < minY) minY = process.Y;
                        if (process.Y > maxY) maxY = process.Y;

                        Point n; int idx;
                        //Put neighbors in stack
                        if (process.X - 1 >= 0)
                        {
                            n = new Point(process.X - 1, process.Y); idx = n.X + nWidth * n.Y;
                            if (!visited[idx]) { visited[idx] = true; toBeProcessed.Push(n); }
                        }

                        if (process.X + 1 < nWidth)
                        {
                            n = new Point(process.X + 1, process.Y); idx = n.X + nWidth * n.Y;
                            if (!visited[idx]) { visited[idx] = true; toBeProcessed.Push(n); }
                        }

                        if (process.Y - 1 >= 0)
                        {
                            n = new Point(process.X, process.Y - 1); idx = n.X + nWidth * n.Y;
                            if (!visited[idx]) { visited[idx] = true; toBeProcessed.Push(n); }
                        }

                        if (process.Y + 1 < nHeight)
                        {
                            n = new Point(process.X, process.Y + 1); idx = n.X + nWidth * n.Y;
                            if (!visited[idx]) { visited[idx] = true; toBeProcessed.Push(n); }
                        }
                    }

                    rec.Add(new Rectangle(minX, minY, maxX - minX + 1, maxY - minY + 1));
                }

                p1 += bytesPerPixel;
                p2 += bytesPerPixel;
            }

            base1 += stride1;
            base2 += stride2;
        }


    bmp.UnlockBits(bmData1);
    bmp2.UnlockBits(bmData2);

    return rec;
}
Nico Schertler
  • 32,049
  • 4
  • 39
  • 70
  • nice. .seems like like that's what I need. Is it worming? Did you try to see the results with the 2 pictures? – Slashy Aug 20 '15 at 08:39
  • yea just tried it.. 5 second for the entire method.. i was looking for somthing really fast... do you think we can improve the perofamance of this? tried using `Parralel` loop but it will be a little problematic with the stack.. – Slashy Aug 20 '15 at 16:02
  • You can get even faster (about 50%) if you can assume a constant `bytesPerPixel` and hard-code the pixel comparison. Parallelization does not help much for this short period. It even decreases performance in my tests with this code. – Nico Schertler Aug 20 '15 at 16:43
  • so odd... cant find why.. i guess a small typo somewhere :) – Slashy Aug 20 '15 at 17:49
  • Yes, it was a typo (one superfluous `!`). As others already suggested, examining every other pixel speeds up the entire process significantly. – Nico Schertler Aug 20 '15 at 18:06
  • exactly what i wanted! one last thing: im gonaa make it easier even. suppose i want to get only the rectangles which the width and the height is bigger then 5. where that coniditon should be? if we want to conserve more time? – Slashy Aug 20 '15 at 18:39
  • Then you would increase `y` by `5` in the loop (and increase `base1/2` by `5 * stride1/2`. And increase `x` by 5 in the loop and `p1/2` by `5 * bytesPerPixel`. – Nico Schertler Aug 20 '15 at 19:13
  • look up in the post i only increased the `x` and `y` but i didnt understand where to increase the `base1`... see in the post – Slashy Aug 20 '15 at 19:35
  • It's the last thing in the loops. `base1 += 5 * stride1; base2 += 5 * stride2` and `p1 += 5 * bytesPerPixel; p2 += 5 * bytesPerPixel;`. – Nico Schertler Aug 20 '15 at 20:12
  • allright.. updated code.. look at the post.. but im still getting 1x1 tiny rectangles... see maybe im doing somthing wrong? im gonna accept your answer anyway! thanks for everything! @NicoSchertler – Slashy Aug 20 '15 at 20:20
  • These rectangles may occur if the sampling accidentally hits a single different pixel. If you want to filter these rectangles, you should check the dimensions before adding them in the line `rec.Add(...)`. – Nico Schertler Aug 20 '15 at 20:24
  • got it. awesome. you helped me so much.enjoy your rep XDD – Slashy Aug 20 '15 at 20:29
  • may i bother you once again. what did you mean by saying ` examining every other pixel speeds up the entire process significantly`? to not increaes the values of `x` and `y` by 5? and only increase regulary by 1?thanks! – Slashy Aug 21 '15 at 12:46
  • No, always increase by 5. But apparently, the flood fill part takes much more time if there are differences than the rest of the code. – Nico Schertler Aug 22 '15 at 07:33
1

You can achieve this easily using a flood fill segmentation algorithm.

First an utility class to make fast bitmap access easier. This will help to encapsulate the complex pointer-logic and make the code more readable:

class BitmapWithAccess
{
    public Bitmap Bitmap { get; private set; }
    public System.Drawing.Imaging.BitmapData BitmapData { get; private set; }

    public BitmapWithAccess(Bitmap bitmap, System.Drawing.Imaging.ImageLockMode lockMode)
    {
        Bitmap = bitmap;
        BitmapData = bitmap.LockBits(new Rectangle(Point.Empty, bitmap.Size), lockMode, System.Drawing.Imaging.PixelFormat.Format32bppArgb);
    }

    public Color GetPixel(int x, int y)
    {
        unsafe
        {
            byte* dataPointer = MovePointer((byte*)BitmapData.Scan0, x, y);

            return Color.FromArgb(dataPointer[3], dataPointer[2], dataPointer[1], dataPointer[0]);
        }
    }

    public void SetPixel(int x, int y, Color color)
    {
        unsafe
        {
            byte* dataPointer = MovePointer((byte*)BitmapData.Scan0, x, y);

            dataPointer[3] = color.A;
            dataPointer[2] = color.R;
            dataPointer[1] = color.G;
            dataPointer[0] = color.B;
        }
    }

    public void Release()
    {
        Bitmap.UnlockBits(BitmapData);
        BitmapData = null;
    }

    private unsafe byte* MovePointer(byte* pointer, int x, int y)
    {
        return pointer + x * 4 + y * BitmapData.Stride;
    }
}

Then a class representing a rectangle containing different pixels, to mark them in the resulting image. In general this class can also contain a list of Point instances (or a byte[,] map) to make indicating individual pixels in the resulting image possible:

class Segment
{
    public int Left { get; set; }
    public int Top { get; set; }
    public int Right { get; set; }
    public int Bottom { get; set; }
    public Bitmap Bitmap { get; set; }

    public Segment()
    {
        Left = int.MaxValue;
        Right = int.MinValue;
        Top = int.MaxValue;
        Bottom = int.MinValue;
    }
};

Then the steps of a simple algorithm are as follows:

  • find different pixels
  • use a flood-fill algorithm to find segments on the difference image
  • draw bounding rectangles for the segments found

The first step is the easiest one:

static Bitmap FindDifferentPixels(Bitmap i1, Bitmap i2)
{
    var result = new Bitmap(i1.Width, i2.Height, System.Drawing.Imaging.PixelFormat.Format32bppArgb);
    var ia1 = new BitmapWithAccess(i1, System.Drawing.Imaging.ImageLockMode.ReadOnly);
    var ia2 = new BitmapWithAccess(i2, System.Drawing.Imaging.ImageLockMode.ReadOnly);
    var ra = new BitmapWithAccess(result, System.Drawing.Imaging.ImageLockMode.ReadWrite);

    for (int x = 0; x < i1.Width; ++x)
        for (int y = 0; y < i1.Height; ++y)
        {
            var different = ia1.GetPixel(x, y) != ia2.GetPixel(x, y);

            ra.SetPixel(x, y, different ? Color.White : Color.FromArgb(0, 0, 0, 0));
        }

    ia1.Release();
    ia2.Release();
    ra.Release();

    return result;
}

And the second and the third steps are covered with the following three functions:

static List<Segment> Segmentize(Bitmap blackAndWhite)
{
    var bawa = new BitmapWithAccess(blackAndWhite, System.Drawing.Imaging.ImageLockMode.ReadOnly);
    var result = new List<Segment>();

    HashSet<Point> queue = new HashSet<Point>();
    bool[,] visitedPoints = new bool[blackAndWhite.Width, blackAndWhite.Height];

    for (int x = 0;x < blackAndWhite.Width;++x)
        for (int y = 0;y < blackAndWhite.Height;++y)
        {
            if (bawa.GetPixel(x, y).A != 0
                && !visitedPoints[x, y])
            {
                result.Add(BuildSegment(new Point(x, y), bawa, visitedPoints));
            }
        }

    bawa.Release();

    return result;
}

static Segment BuildSegment(Point startingPoint, BitmapWithAccess bawa, bool[,] visitedPoints)
{
    var result = new Segment();

    List<Point> toProcess = new List<Point>();

    toProcess.Add(startingPoint);

    while (toProcess.Count > 0)
    {
        Point p = toProcess.First();
        toProcess.RemoveAt(0);

        ProcessPoint(result, p, bawa, toProcess, visitedPoints);
    }

    return result;
}

static void ProcessPoint(Segment segment, Point point, BitmapWithAccess bawa, List<Point> toProcess, bool[,] visitedPoints)
{
    for (int i = -1; i <= 1; ++i)
    {
        for (int j = -1; j <= 1; ++j)
        {
            int x = point.X + i;
            int y = point.Y + j;

            if (x < 0 || y < 0 || x >= bawa.Bitmap.Width || y >= bawa.Bitmap.Height)
                continue;

            if (bawa.GetPixel(x, y).A != 0 && !visitedPoints[x, y])
            {
                segment.Left = Math.Min(segment.Left, x);
                segment.Right = Math.Max(segment.Right, x);
                segment.Top = Math.Min(segment.Top, y);
                segment.Bottom = Math.Max(segment.Bottom, y);

                toProcess.Add(new Point(x, y));
                visitedPoints[x, y] = true;
            }
        }
    }
}

And the following program given your two images as arguments:

static void Main(string[] args)
{
    Image ai1 = Image.FromFile(args[0]);
    Image ai2 = Image.FromFile(args[1]);

    Bitmap i1 = new Bitmap(ai1.Width, ai1.Height, System.Drawing.Imaging.PixelFormat.Format32bppArgb);
    Bitmap i2 = new Bitmap(ai2.Width, ai2.Height, System.Drawing.Imaging.PixelFormat.Format32bppArgb);

    using (var g1 = Graphics.FromImage(i1))
    using (var g2 = Graphics.FromImage(i2))
    {
        g1.DrawImage(ai1, Point.Empty);
        g2.DrawImage(ai2, Point.Empty);
    }

    var difference = FindDifferentPixels(i1, i2);
    var segments = Segmentize(difference);

    using (var g1 = Graphics.FromImage(i1))
    {
        foreach (var segment in segments)
        {
            g1.DrawRectangle(Pens.Red, new Rectangle(segment.Left, segment.Top, segment.Right - segment.Left, segment.Bottom - segment.Top));
        }
    }

    i1.Save("result.png");

    Console.WriteLine("Done.");
    Console.ReadKey();
}

produces the following result:

enter image description here

As you can see there are more differences between the given images. You can filter the resulting segments with regard to their size for example to drop the small artefacts. Also there is of course much work to do in terms of error checking, design and performance.

One idea is to proceed as follows:

1) Rescale images to a smaller size (downsample)

2) Run the above algorithm on smaller images

3) Run the above algorithm on original images, but restricting yourself only to rectangles found in step 2)

This can be of course extended to a multi-level hierarchical approach (using more different image sizes, increasing accuracy with each step).

BartoszKP
  • 34,786
  • 15
  • 102
  • 130
0

Ah an algorithm challenge. Like! :-)

There are other answers here using f.ex. floodfill that will work just fine. I just noticed that you wanted something fast, so let me propose a different idea. Unlike the other people, I haven't tested it; it shouldn't be too hard and should be quite fast, but I simply don't have the time at the moment to test it myself. If you do, please share the results. Also, note that it's not a standard algorithm, so there are probably some bugs here and there in my explanation (and no patents).

My idea is derived from the idea of mean adaptive thresholding but with a lot of important differences. I cannot find the link from wikipedia anymore or my code, so I'll do this from the top of my mind. Basically you create a new (64-bit) buffer for both images and fill it with:

f(x,y) = colorvalue + f(x-1, y) + f(x, y-1) - f(x-1, y-1)
f(x,0) = colorvalue + f(x-1, 0)
f(0,y) = colorvalue + f(0, y-1)

The main trick is that you can calculate the sum value of a portion of the image fast, namely by:

g(x1,y1,x2,y2) = f(x2,y2)-f(x1-1,y2)-f(x2,y1-1)+f(x1-1,y1-1)

In other words, this will give the same result as:

result = 0;
for (x=x1; x<=x2; ++x) 
  for (y=y1; y<=y2; ++y)    
    result += f(x,y)

In our case this means that with only 4 integer operations this will get you some unique number of the block in question. I'd say that's pretty awesome.

Now, in our case, we don't really care about the average value; we just care about some sort-of unique number. If the image changes, it should change - simple as that. As for colorvalue, usually some gray scale number is used for thresholding - instead, we'll be using the complete 24-bit RGB value. Because there are only so few compares, we can simply scan until we find a block that doesn't match.

The basic algorithm that I propose works as follows:

for (y=0; y<height;++y)
    for (x=0; x<width; ++x)
       if (src[x,y] != dst[x,y])
          if (!IntersectsWith(x, y, foundBlocks))
              FindBlock(foundBlocks);

Now, IntersectsWith can be something like a quad tree of if there are only a few blocks, you can simply iterate through the blocks and check if they are within the bounds of the block. You can also update the x variable accordingly (I would). You can even balance things by re-building the buffer for f(x,y) if you have too many blocks (more precise: merge found blocks back from dst into src, then rebuild the buffer).

FindBlocks is where it gets interesting. Using the formula for g that's now pretty easy:

int x1 = x-1; int y1 = y-1; int x2 = x; int y2 = y; 
while (changes)
{
    while (g(srcimage,x1-1,y1,x1,y2) == g(dstimage,x1-1,y1,x1,y2)) { --x1; }
    while (g(srcimage,x1,y1-1,x1,y2) == g(dstimage,x1,y1-1,x1,y2)) { --y1; }
    while (g(srcimage,x1,y1,x1+1,y2) == g(dstimage,x1,y1,x1+1,y2)) { ++x1; }
    while (g(srcimage,x1,y1,x1,y2+1) == g(dstimage,x1,y1,x1,y2+1)) { ++y1; }
}

That's it. Note that the complexity of the FindBlocks algorithm is O(x + y), which is pretty awesome for finding a 2D block IMO. :-)

As I said, let me know how it turns out.

atlaste
  • 30,418
  • 3
  • 57
  • 87
  • haha not a challenge actually .. just helping a little friend ;) thanks for your answer, and yea I do need a fast and efficient method cut basically I need to process up to 2m pixels constantly! I would like you to add more explanation( of course only when you can and have free time) about your suggestion.i think I'm starting to get it. But not 100% .I'm only few months with the image processing subject. .. thanks in advance! – Slashy Aug 19 '15 at 09:48
  • @Slashy Basically `g(...)` calculates the sum of a rectangle. I probably made some errors in the calculations, I remember that I should use `-1` for x and y. Simply try implementing `f` and `g` in Excel on a small grid with random numbers and see the magic happen. – atlaste Aug 19 '15 at 09:51
  • @Slashy Again, they represent the *sum* of the pixel values. Which is sort-of unique. Have you tried the Excel tip? – atlaste Aug 19 '15 at 10:38
  • I'm not at home yet. . A vacation XD .coming back tomorrow. The sum means the a+b+g+r? – Slashy Aug 19 '15 at 10:47
  • That's a bad excuse, there's always Google Calc. Just try it; there's no sense in explaining these details otherwise. – atlaste Aug 19 '15 at 10:49
  • 1
    Cool idea. I think that's called an "integral image". One suggestion: Instead of calculating two integral images, I would calculate one integral image of the absolute differences of the two images. – Niki Aug 19 '15 at 19:35
  • @nikie Integral images are with 1 component normally, but yes that was what I was looking for (+1 for that). And that depends: I guessed he's using it to create some kind of remote desktop - if that's the case, you might as well swap around the buffers. If not, I'd say it's better to simply XOR the images and use that. As for the naming part, I hereby dub this "DeBruijn image segmentation" :-) Use it in whatever way you see fit :-) – atlaste Aug 19 '15 at 19:47
  • Hm, I never thought about why he would want that. But if it's for a "homegrown VNC", I'd do something much simpler: regularly calculate a CRC checksum for every 8x8 block. Then re-transmit 8x8 blocks every time the checksum changes. Unless every change is exactly rectangular, this is probably more efficient than re-transmitting the whole boundary rectangle, too. – Niki Aug 20 '15 at 16:29
  • @nikie if you're interested, I've written down a complete way to do this last week on this URL: http://stackoverflow.com/questions/31543940/c-sharp-screen-transfer-over-socket-efficient-improve-ways/31603486#31603486 . It was pretty basic, if you XOR images, most of it will be a 0, so if you compress that there won't be much left afterwards. Still, it's better not to send anything if you don't have to, so from that perspective I don't think you can beat a simple segmentation algorithm and send those coordinates per changed segment (in terms of compression). – atlaste Aug 21 '15 at 05:37
  • @nikie do you think it would be more simple and fast? to summ the rgb values? what about using `memcmp`? to compare the bytes? – Slashy Aug 21 '15 at 09:59
  • @atlaste the complexabilty sounds awesome! but would i get the bound from this? i'll show you what i've done so far from the parts i understood.. haha. you must be a veryy smart person.. :) – Slashy Aug 21 '15 at 10:07
  • @Slashy: (Most of my performance tuning experience is from C++, so this might be different for C#): Good ingredients for fast code are: no (non-inline) function calls and no conditionals (branch prediction!) and as few memory reads and writes as possible. Any compare & rectangle find code will have to read pixels from two images and then branch depending on the result. To calculate a CRC checksum you only need to read pixel values from *one* image and have no branches at all. So I would expect this to be faster. – Niki Aug 21 '15 at 14:16
  • @nikie Well you're right and wrong, both in C++ and C# :-) In C# the rule of thumb is: avoid random access memory/branches and code with as few instructions as possible. In C++, it's mostly the former two. Mike Acton has a few good videos about it for C++. You're right in the sense that basically you'll find that you want to avoid 'random access memory', branches and memory operations - and process stuff in byte order. Sequential scans from 2 images are fine, in 1 image is even better - but comparing the hashes and processing an 8x8 grid will eat up your performance due to cache line fills. – atlaste Aug 21 '15 at 14:24
  • @atlaste: I doubt that writing one 32bit hash value to an aligned memory location for one in 64 pixels will eat up much performance. If you're worried, use a larger block size or store the CRC checksums in a SIMD register until you can write it to memory. – Niki Aug 21 '15 at 14:37
  • @nikie no that won't eat up your performance - the fact that you're using 8 y values for each block will eat it up, because that's unaligned memory access. I suppose you can fix that of course by keeping track of multiple crc checks at once... Anyways, best way to do these kinds of things IMO is to simply try them out and while you're not satisfied, put a breakpoint in the inner loop and check the assembler... at least that's how I usually do it. – atlaste Aug 21 '15 at 15:04
  • I still don't get it. Why would the access be unaligned? The image has 4 byte per pixel, so the offset's `y*stride + x*8*4` - that should always be aligned to a 32 byte boundary, right? So for each 8x8 block, I'd access 8 full cache lines. Don't see how you could get away with less if you want to check every pixel. Either way, you're right of course, this is splitting hairs: the only right way optimize that is using a good profiler. – Niki Aug 21 '15 at 15:44