c# screen sharing Efficiency

Question

I'm trying to optimize my screen sharing app. I've already used a few ways to make it faster and stable, such as only sending the deltas between two frames, and using Gzip to compress the data.

This is my client code:

    private void Form1_Load(object sender, EventArgs e)
    {

        Thread th = new Thread(startSend);
        th.Start(); 
       }


    private void startSend()
    {
        Bitmap curr;
        Bitmap diff;
        encoderParams.Param[0] = qualityParam;

       Bitmap pre = screenshot();

        bmpBytes = imageToByteArray(pre);

        SendVarData(handler, bmpBytes);

        while (true)
        {


            curr= screenshot();

        diff= Difference(pre, curr);


            bmpBytes = imageToByteArray(diff);


            SendVarData(handler, bmpBytes);

            pre = curr;

        }
    }

Screenshot:

public Bitmap screenshot()
    {

        Bitmap screenshot = new Bitmap(SystemInformation.VirtualScreen.Width,
                              SystemInformation.VirtualScreen.Height,
                              PixelFormat.Format24bppRgb);
        Graphics screenGraph = Graphics.FromImage(screenshot);
        screenGraph.CopyFromScreen(0,
                                   0,
                                   0,
                                   0,
                                   SystemInformation.VirtualScreen.Size,
                                   CopyPixelOperation.SourceCopy);

        return screenshot;

    }

The Difference method:

 public  Bitmap Difference(Bitmap bmp0, Bitmap bmp1)
    {
        Bitmap bmp2;

        int Bpp = 3;

       bmp2 = new Bitmap(bmp0.Width, bmp0.Height, bmp0.PixelFormat);

        var bmpData0 = bmp0.LockBits(
                        new Rectangle(0, 0, bmp0.Width, bmp0.Height),
                        ImageLockMode.ReadOnly, bmp0.PixelFormat);
        var bmpData1 = bmp1.LockBits(
                        new Rectangle(0, 0, bmp1.Width, bmp1.Height),
                        ImageLockMode.ReadOnly, bmp1.PixelFormat);
        var bmpData2 = bmp2.LockBits(
                        new Rectangle(0, 0, bmp2.Width, bmp2.Height),
                        ImageLockMode.ReadWrite, bmp2.PixelFormat);

        bmp0.UnlockBits(bmpData0);
        bmp1.UnlockBits(bmpData1);
        bmp2.UnlockBits(bmpData2);
        int len = bmpData0.Height * bmpData0.Stride;

     //   MessageBox.Show(bmpData0.Stride.ToString());
        bool changed=false;

        byte[] data0 = new byte[len];
        byte[] data1 = new byte[len];
        byte[] data2 = new byte[len];
        Marshal.Copy(bmpData0.Scan0, data0, 0, len);
        Marshal.Copy(bmpData1.Scan0, data1, 0, len);
        Marshal.Copy(bmpData2.Scan0, data2, 0, len);

        for (int i = 0; i < len; i += Bpp)
        {


               changed = ((data0[i] != data1[i])
                             || (data0[i + 1] != data1[i + 1])
                             || (data0[i + 2] != data1[i + 2]));



                // this.Invoke(new Action(() => this.Text = changed.ToString()));

                data2[i] = changed ? data1[i] : (byte)2;   // special markers
                data2[i + 1] = changed ? data1[i + 1] : (byte)3;   // special markers
                data2[i + 2] = changed ? data1[i + 2] : (byte)7;   // special markers
                if (Bpp == 4) data2[i + 3] =
                                 changed ? (byte)255 : (byte)42;  // special markers



        }


     //  this.Invoke(new Action(() => this.Text = changed.ToString()));
        Marshal.Copy(data2, 0, bmpData2.Scan0, len);


        return bmp2;
    }

and the SendVarData function:

int total = 0;
    byte[] datasize;
    private  int SendVarData(Socket s, byte[] data)
    {
       total = 0;
        int size = data.Length;
        int dataleft = size;
        int sent;


        datasize = BitConverter.GetBytes(size);
        sent = s.Send(datasize);


            sent = s.Send(data, total, dataleft, SocketFlags.None);
            total += sent;
            dataleft -= sent;
          //  MessageBox.Show("D");

        return total;
    }

This is the server - I'm just receiving a full image in the beginning, and then just deltas:

    public void startListening()
    {

        prev =  byteArrayToImage(ReceiveVarData(client.Client));

        theImage.Image = prev;
        while (true)
        {

            data = ReceiveVarData(client.Client);

            curr = byteArrayToImage(data) ;

            merge = Merge(prev, curr);

            theImage.Image = merge;

            count++;

            prev = merge;
        }

    }

   public static Bitmap Merge(Bitmap bmp0, Bitmap bmp1)
 {

int Bpp = 3;

Bitmap bmp2 = new Bitmap(bmp0.Width, bmp0.Height, bmp0.PixelFormat);

var bmpData0 = bmp0.LockBits(
                new System.Drawing.Rectangle(0, 0, bmp0.Width, bmp0.Height),
                ImageLockMode.ReadOnly, bmp0.PixelFormat);
var bmpData1 = bmp1.LockBits(
                new System.Drawing.Rectangle(0, 0, bmp1.Width, bmp1.Height),
                ImageLockMode.ReadOnly, bmp1.PixelFormat);
var bmpData2 = bmp2.LockBits(
                new System.Drawing.Rectangle(0, 0, bmp2.Width, bmp2.Height),
                ImageLockMode.ReadWrite, bmp2.PixelFormat);
bmp0.UnlockBits(bmpData0);
bmp1.UnlockBits(bmpData1);
bmp2.UnlockBits(bmpData2);

int len = bmpData0.Height * bmpData0.Stride;
byte[] data0 = new byte[len];
byte[] data1 = new byte[len];
byte[] data2 = new byte[len];
Marshal.Copy(bmpData0.Scan0, data0, 0, len);
Marshal.Copy(bmpData1.Scan0, data1, 0, len);
Marshal.Copy(bmpData2.Scan0, data2, 0, len);

for (int i = 0; i < len; i += Bpp)
{

        bool toberestored = (data1[i] != 2 && data1[i + 1] != 3 &&
                                data1[i + 2] != 7 && data1[i + 2] != 42);
        if (toberestored)
        {
            data2[i] = data1[i];
            data2[i + 1] = data1[i + 1];
            data2[i + 2] = data1[i + 2];
            if (Bpp == 4) data2[i + 3] = data1[i + 3];
        }
        else
        {
            data2[i] = data0[i];
            data2[i + 1] = data0[i + 1];
            data2[i + 2] = data0[i + 2];
            if (Bpp == 4) data2[i + 3] = data0[i + 3];
        }


}

Marshal.Copy(data2, 0, bmpData2.Scan0, len);

return bmp2;

}

I think it's coded fine, but I'm still unable to get more than 6~7fps (of 8kb-100kb) when running on 2 different computers with a fast and stable internet connection, and a maximum of 11fps when running both client and server on the same computer. I think it's because of the complexity of the delta and merging algorithms, but i dont know.

I would very appreciate if anyone could suggest how could optimize it further.

Answer 1

You may organize data2 in a different manner to send less data. The below "compression" algorithm is very basic but will provide an improved compression compared to your implelmentation.

Search for differences, by identifying start and end of consecutive differences. When you find an interval where all pixels are different, store the length of identical data before that interval using 2 bytes, then store the number of consecutive differences, finally write 3 RGB bytes for each different pixel.

In case of 65535 different pixels, block the max interval length to 65535 and after storing the interval values. The next difference interval starting just after the stored interval, the identical count for next interval will be 0.

In case of 65535 identical pixels, just write $FFFF followed by $0000 indicating an empty sequence of different pixels.

Clarification: What I mean by "identifying start and end of consecutive differences" ?

In the above example, letters identify colors, i.e W(white), P(Pink), O(Orange): The word "identical" refers to a comparison between data 0 and data1 (not comparing data1[i] with data1[i-1]).

Data0 = WWPWWOWOOOWWOOOPP
Data1 = WWPWPWP OOOWWPP OPP

You have 4 identical pixels (WWPW) followed by an interval (length 3) starting on the 4th character and ending on the 6th where all pixels are different. Then five identical pixels, followed by a new interval with 2 differences. At the end, a few common pixels.

The output for data2 will be (the text in parenthesis are not part of the buffer and explains the previous buffer values :

04 00 (4 identical pixels) 03 00 (3 different pixels coded in next 9 bytes) 
Pr Pg Pb (3 bytes RGB code for P) Wr Wg Wb (RGB code for W) Pr Pg Pb (RGB code for P)
05 00 (5 identical pixels) 03 00 (2 different pixels coded in next 6 bytes) 
Pr Pg Pb (RGB code for P) ...

Answer 2

The code to write differences would look like to the following. I let you build the code adressing the opposite action, i.e. reading differences to update the previous image.

    Data2Index     = 0 ; // next index for additions in data2 ; 
    int idcount   = 0 ;
    int diffstart = -1 ;
    int diffstart = -1 ;
    for (int i = 0; i < len; i += Bpp)
    {
       changed = ((data0[i] != data1[i])
               || (data0[i + 1] != data1[i + 1])
               || (data0[i + 2] != data1[i + 2]));
       if (!changed)
   {
     if (idcount==ushort.MaxValue) 
     { // still identical, but there is a limitation on count 
           // write to data2 the identical count + differencecount equals to 0
           AddIdCountDiffCountAndDifferences(idcount,0,0) ;
           idcount = 0 ;
         }
         if (diffstart>0)
         { // after 0 or more identical values, a change was found
           // write to data2 the identical count + difference count + different pixels
           AddIdCountDiffCountAndDifferences(idcount,diffcount,diffstart) ;
       idcount = 0  ;
           diffcount= 0 ; 
           diffstart=-1 ;
     }
         else identicalcount++ ; // still identical, continue until difference found
       }
       else if (diffstart<0) 
       { // a difference is found after a sequence of identical pixels, store the index of first difference  
         diffstart=i ; diffcount=1 ; 
       }
       else 
       { // different pixel follows another difference (and limitation not reached)
         if (diffcount<ushort.MaxVakue) diffcount++ ;
       }
       else  
       { // limitation reached, i.e. diffcount equals 65535 
         AddIdCountDiffCountAndDifferences(0,diffcount,diffstart) ;
         diffstart+=diffcount ;
         diffcount=0 ;
       }

The procedure used to fill data2 here:

   private int Data2Index = 0 ; // to be reset before       

   private void AddIdCountDiffCountAndDifferences(int idcount,int diffcount,int diffstart)
   {
     data2[Data2Index++]=(byte)(idcount        && 0xFF) ; // low byte of the int
     data2[Data2Index++]=(byte)(idcount   >> 8 && 0xFF) ; // second byte of the int 
     data2[Data2Index++]=(byte)(diffcount      && 0xFF) ; // low byte of the int
     data2[Data2Index++]=(byte)(diffcount >> 8 && 0xFF) ; // second byte of the int 
     for (int i=0;i<diffcount;i++)
     {
        data2[Data2Index++]=data1[diffstart+Bpp*i  ] ;
        data2[Data2Index++]=data1[diffstart+Bpp*i+1] ;
        data2[Data2Index++]=data1[diffstart+Bpp*i+2] ;
      }
    }