MODI MiSelectRects gets coordinates wrong

Question

I have a Windows Form app that, when executed, launches Firefox, grabs the process and handle of the window, and does a screen capture of Firefox, saves it to disk (temp.bmp) and calls ProcessGetWindow. I'm basically using MiSelectRects in MODI to capture the rectangle around the word that I'm looking for, and then I use AutoIT to mouse click on the word.

The problem is that my coordinates are off by about 10 pixels from the top.

Any ideas what might be wrong? Here's the function that does the processing. I have commented out the AutoIT processing, and I'm just debugging with a MessageBox to show me the actual coordinates. I then confirm with AutoIT's Window Info tool and it's definitely off... am I doing something wrong or is there something screwed up with MODI?

public void ProcessGetWindow(Bitmap image)
        {           
            Document modiDoc = null;
            MiDocSearch modiSearch = null;
            IMiSelectableItem modiTextSel = null;
            MiSelectRects modiSelectRects = null;
            MiSelectRect modiSelectRect = null;
            MiRects modiRects = null;
            int intSelInfoPN;
            string intSelInfoTop;
            int intSelInfoBottom;
            string intSelInfoLeft;
            int intSelInfoRight;            

            // Load an existing image file.
            modiDoc = new Document();
            modiDoc.Create(@"C:\\temp.bmp");

            // Perform OCR.
            modiDoc.Images[0].OCR();

            // Search for the selected word.
            modiSearch = new MiDocSearch();
            modiSearch.Initialize(modiDoc, "Click Me", 0, 0, false, false);
            modiSearch.Search(null, ref modiTextSel);       

            try
            {          
                modiSelectRects = modiTextSel.GetSelectRects();
            }
            catch (COMException)
            {
                MessageBox.Show("Me thinks that the OCR didn't work right!");
            }

            foreach (MiSelectRect mr in modiSelectRects)
            {
                //intSelInfoPN = mr.PageNumber.ToString();
                intSelInfoTop = mr.Top.ToString();
                //intSelInfoBottom = mr.Bottom;
                intSelInfoLeft = mr.Left.ToString();
                //intSelInfoRight = mr.Right;

                /*AutoItX3 auto = new AutoItX3();
                auto.AutoItSetOption("MouseCoordMode", 2);
                auto.MouseClick("", intSelInfoLeft, intSelInfoTop, 1, 80);*/

                MessageBox.Show("Coordinates: " + intSelInfoLeft + ", " + intSelInfoTop, "Coordinates", MessageBoxButtons.OK);            
            }

            //string textResult = modiTextSel.Text;

            //MessageBox.Show(textResult, "Search Results", MessageBoxButtons.OK);

            // Close this dialog.
            Application.Exit();
        }

Answer 1

I am using the same program to find the location.

int centerwidth = (intSelInfoRight - intSelInfoLeft)/2;
                centerwidth = intSelInfoLeft + centerwidth;
                 int centerheight = (intSelInfoBottom - intSelInfoTop)/2;
                 centerheight = centerheight + intSelInfoTop;

u can find the exact middle point of the text using it.

But this programs always gives the location of the 1st occurence of the word and not for the next occurences. Please let me know how to find the location of the text at all occurences.

Answer 2

 MODI.Document modiDoc = null;
    MODI.MiDocSearch modiSearch = null;
    MODI.IMiSelectableItem modiTextSel = null;
    MODI.MiSelectRects modiSelectRects = null;
    MODI.MiSelectRect modiSelectRect = null;
    MODI.MiRects modiRects = null;
    int intSelInfoPN;
    int intSelInfoTop;
    int intSelInfoBottom;
    int intSelInfoLeft;
    int intSelInfoRight;

    // Load an existing image file.
    modiDoc = new MODI.Document();
    modiDoc.Create(@"C:\Users\h117953\Desktop\OCR\1.jpg");

    // Perform OCR.
    //modiDoc.Images[0].OCR();
    //MODI.Image image = (MODI.Image)modiDoc.Images[0];
    modiDoc.OCR(MiLANGUAGES.miLANG_ENGLISH);
    MODI.Image modiImage = (modiDoc.Images[0] as MODI.Image);


    //string ocrtext = @"C:\Users\h117953\Desktop\OCR\Sample.txt";

    //File.WriteAllText(ocrtext, modiImage.Layout.Text);

    // Search for the selected word.
    //int wordindex
    modiSearch = new MODI.MiDocSearch();
    //date to search 
    modiSearch.Initialize(modiDoc, "Deer", 0, 2, false, false);
    modiSearch.Search(null, ref modiTextSel);
    if (modiTextSel == null)
    {
        Response.Write("\nText not found \n");


    }
    else
    {
        Response.Write("\nText is found \n");
        try
        {
            modiSelectRects = modiTextSel.GetSelectRects();
        }
        catch (Exception)
        {
            Response.Write("Me thinks that the OCR didn't work right!");
        }

        int centerwidth = 0;
        int centerheight = 0;

        foreach (MODI.MiSelectRect mr in modiSelectRects)
        {  
            //intSelInfoPN = mr.PageNumber.ToString();
            intSelInfoTop = mr.Top;
            intSelInfoBottom = mr.Bottom;
            intSelInfoLeft = mr.Left;
            intSelInfoRight = mr.Right;


            // MessageBox.Show("Coordinates: " + intSelInfoLeft + ", " + intSelInfoTop, "Coordinates", MessageBoxButtons.OK);
            //  MessageBox.Show("Coordinates: " + intSelInfoRight + ", " + intSelInfoBottom, "Coordinates", MessageBoxButtons.OK);
            centerwidth = (intSelInfoRight - intSelInfoLeft) / 2;
            centerwidth = intSelInfoLeft + centerwidth;
            centerwidth = (intSelInfoBottom - intSelInfoTop) / 2;
            centerheight = centerheight + intSelInfoTop;

            //MessageBox.Show("Coordinates: " + centerwidth + ", " + centerheight, "Coordinates", MessageBoxButtons.OK);
            Response.Write("the Widht and Height co-ordinates are (Width,Height)= ({0},{1})" + centerwidth + ","+ centerheight);



        }

Answer 3

I'm not familiar with the tools presented, but from what I read the GetSelectRects function returns a bounding rectangle, that is the smallest rectangle that contains the whole selection, in this case the word you searched for. I believe what happens is that you're clicking the corner of the bounding rectangle instead of in the middle, where the word is.

Calculate the co-ordinates for the center of the rectangle and try clicking that:

int height = mr.Bottom - mr.Top;
int width = mr.Right - mr.Left;

AutoItX3 auto = new AutoItX3();
auto.AutoItSetOption("MouseCoordMode", 2);
auto.MouseClick("", width/2, height/2, 1, 80);