Using UI Automation to read Internet Explorer DOM from a pane(Window)

Question

I am using UISpy.exe (Windows tool) to select UI Elements that also works with IE. However, for some IE Elements, all properties are not loaded in the automation framework or they have weird values with which they can not be uniquely and efficiently identified when looked for(like in the figure below, Name property).

The same properties that are available here are also available using the Windows Automation API Framework. I want to know if there is way, using this or any other library, to access the DOM of a selected window or pane. I know of one other application (UIExplorer from UI Path) that works this way but I can't figure how.

Here is an example of what UIExplorer's selector for the same element looks like. See how they have access to the DOM where as UISpy.exe is only displaying elements.

score 1 · Answer 1 · answered May 17 '18 at 19:23

you can try this:

using System;
using System.Runtime.InteropServices;
using System.Windows.Automation;
using mshtml;

namespace ConsoleApp1
{
    class Program
    {
        #region Native
        [DllImport("user32", EntryPoint = "RegisterWindowMessageA", CharSet = CharSet.Ansi, SetLastError = true, ExactSpelling = true)]
        private static extern int RegisterWindowMessage(string lpString);

        [DllImport("user32", EntryPoint = "SendMessageTimeoutA", CharSet = CharSet.Ansi, SetLastError = true, ExactSpelling = true)]
        private static extern int SendMessageTimeout(IntPtr hWnd, int msg, int wParam, int lParam, int fuFlags, int uTimeout, ref int lpdwResult);

        [DllImport("oleacc", CharSet = CharSet.Ansi, SetLastError = true, ExactSpelling = true)]
        private static extern int ObjectFromLresult(int lResult, ref Guid riid, int wParam, ref IHTMLDocument2 ppvObject);

        private const int SMTO_ABORTIFHUNG = 2;
        #endregion Native

        static void Main(string[] args)
        {
            var firstIeWindow = GetFirstIeWindow();
            var firstIeTab = GetFirstIeTab(firstIeWindow);
            var ieDom = GetIeDom(new IntPtr(firstIeTab.Current.NativeWindowHandle));
            var firstInput = GetFirstInputElement(ieDom);

            Console.WriteLine($"#{firstInput.id}[name={firstInput.name}] = {firstInput.value}");
        }

        private static AutomationElement GetFirstIeWindow()
        {
            var firstIeWindow = AutomationElement.RootElement.FindFirst(TreeScope.Children, new AndCondition(
                                    new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Window),
                                    new PropertyCondition(AutomationElement.ClassNameProperty, "IEFrame")))
                                ?? throw new Exception("IEFrame not found");
            return firstIeWindow;
        }

        private static AutomationElement GetFirstIeTab(AutomationElement ieWindow)
        {
            var frameTab = ieWindow.FindFirst(TreeScope.Children, new AndCondition(
                               new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Pane),
                               new PropertyCondition(AutomationElement.ClassNameProperty, "Frame Tab")))
                           ?? throw new Exception("Frame Tab not found");

            var tabWindow = frameTab.FindFirst(TreeScope.Children, new AndCondition(
                                new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Pane),
                                new PropertyCondition(AutomationElement.ClassNameProperty, "TabWindowClass")))
                            ?? throw new Exception("TabWindowClass not found");

            var ieServer = tabWindow.FindFirst(TreeScope.Descendants, new AndCondition(
                               new PropertyCondition(AutomationElement.ControlTypeProperty, ControlType.Pane),
                               new PropertyCondition(AutomationElement.ClassNameProperty, "Internet Explorer_Server")))
                           ?? throw new Exception("Internet Explorer_Server not found");

            return ieServer;
        }

        private static DispHTMLDocument GetIeDom(IntPtr hWnd)
        {
            var lRes = 0;
            var lMsg = RegisterWindowMessage("WM_HTML_GETOBJECT");
            SendMessageTimeout(hWnd, lMsg, 0, 0, SMTO_ABORTIFHUNG, 1000, ref lRes);
            if (lRes == 0) return null;

            IHTMLDocument2 ieDomFromhWnd = null;
            var iidIhtmlDocument2 = new Guid("626FC520-A41E-11CF-A731-00A0C9082637");
            var hr = ObjectFromLresult(lRes, ref iidIhtmlDocument2, 0, ref ieDomFromhWnd);
            if (hr != 0) throw new COMException($"{nameof(ObjectFromLresult)} has thrown an exception", hr);
            return ieDomFromhWnd as DispHTMLDocument ?? throw new Exception("IE DOM not found");
        }

        private static DispHTMLInputElement GetFirstInputElement(DispHTMLDocument ieDom)
        {
            var firstInput = (ieDom.body as DispHTMLBody)?.getElementsByTagName("input").item(0) as DispHTMLInputElement
                             ?? throw new Exception("Input element not found");
            return firstInput;
        }
    }
}

Using UI Automation to read Internet Explorer DOM from a pane(Window)

1 Answers1