Is it possible to click inside a self made web app?

Question

I created a PyQt5 web app that navigates to https://www.google.com/ with the following code :

from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
from PyQt5.QtGui import *
from PyQt5.QtWebEngineWidgets import *

import sys

class MainWindow(QMainWindow):

    def __init__(self, *args, **kwargs):
        super(MainWindow, self).__init__(*args, **kwargs)

        self.setWindowTitle("My Window")

        self.browser = QWebEngineView()
        self.browser.setUrl( QUrl("www.google.com") )
        self.setCentralWidget(self.browse) 

app = QApplication(sys.argv)

window = MainWindow()
window.show()

app.exec()

The idea is that I want to manipulate clicks inside the application, therefore on the google site, for example: moving the mouse to certain locations(x, y), without influencing the PC itself, like pyautogui does, meaning that I can run a code that randomly clicks inside the web app while I am able to do normal things on the PC, without affecting the mouse.

I will be glad to know if this is possible or not. And if it is, how?

Note : I don't really have experience in python

While not exactly what you are asking for, you could also just have the selenium code run in a vm. Which would free up your mouse. — mrfreester, May 01 '19 at 15:58
@mrfreester What the OP requires is to be able to emulate the click events (and other events) in the custom browser made with QWebEngineView — eyllanesc, May 01 '19 at 16:15

score 1 · Answer 1 · answered May 01 '19 at 14:47

Unless it is an embedded app, you can always parse the html, search for the button and "click" the button via either requesting the path that it would go or by a javascript parser.

After the code api would look something like

parsed_response.find(my_button).click()

If you don't insist on writing your own, there is already one made, selenium.

To show a little example

from selenium import webdriver
from selenium.webdriver.common.keys import Keys

driver = webdriver.Chrome()
driver.get("www.website.com")
login_input = driver.find_element_by_id(id)
login_input.send_keys(my_login)
login_input.send_keys(Keys.ENTER)
# Or you can do something like
# submit_button = driver.find_element_by_id(submitbutton)
# submit_button.click()

The power of selenium is huge but I always add this as a note when I answer questions about web-scraping. Please do check /robots.txt before actually using that site with your bot.

eyllanesc · Accepted Answer · 2019-05-01T16:11:13.170

You can emulate the click by sending QMouseEvent, in the case of QWebEngineView the click should be issued to an internal widget that is part of the private Qt API but with a small logic can be accessed: using findChildren and the QMetaObject.

In the following example, click on position 400, 200 which in my case is a google banner that will open respective information.

import sys
from PyQt5 import QtCore, QtGui, QtWidgets, QtWebEngineWidgets


class MainWindow(QtWidgets.QMainWindow):
    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)

        self.setWindowTitle("My Window")

        self.browser = QtWebEngineWidgets.QWebEngineView()
        self.browser.setUrl(QtCore.QUrl("https://www.google.com/"))
        self.setCentralWidget(self.browser)
        self.browser.loadFinished.connect(self.on_loadFinished)

    @QtCore.pyqtSlot(bool)
    def on_loadFinished(self, ok):
        if not ok:
            return
        w = None
        for child in self.browser.findChildren(QtWidgets.QWidget):
            if (
                child.metaObject().className()
                == "QtWebEngineCore::RenderWidgetHostViewQtDelegateWidget"
            ):
                w = child
                break
        if w is not None:
            self.emulate_click(w, QtCore.QPoint(400, 200))

    def emulate_click(self, widget, pos):
        event_press = QtGui.QMouseEvent(
            QtCore.QEvent.MouseButtonPress,
            pos,
            QtCore.Qt.LeftButton,
            QtCore.Qt.LeftButton,
            QtCore.Qt.NoModifier,
        )
        QtCore.QCoreApplication.postEvent(widget, event_press)
        event_release = QtGui.QMouseEvent(
            QtCore.QEvent.MouseButtonRelease,
            pos,
            QtCore.Qt.LeftButton,
            QtCore.Qt.LeftButton,
            QtCore.Qt.NoModifier,
        )
        QtCore.QCoreApplication.postEvent(widget, event_release)


if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    w = MainWindow()
    w.resize(640, 480)
    w.show()
    sys.exit(app.exec_())

If you have more information such as id or xpath you can click using javascript:

import sys
from PyQt5 import QtCore, QtGui, QtWidgets, QtWebEngineWidgets


class MainWindow(QtWidgets.QMainWindow):
    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)

        self.setWindowTitle("My Window")

        self.browser = QtWebEngineWidgets.QWebEngineView()
        self.browser.setUrl(QtCore.QUrl("https://www.google.com/"))
        self.setCentralWidget(self.browser)
        self.browser.loadFinished.connect(self.on_loadFinished)

    @QtCore.pyqtSlot(bool)
    def on_loadFinished(self, ok):
        if not ok:
            return
        xpath = r"//body[@id='gsr']/div[@id='viewport']/div[@id='main']/span[@id='body']/center/div[@id='lga']/div[@id='hplogo']/a/img[1]"
        s = (
            """
            (function() {
                var banner = document.evaluate("%s", document, null, XPathResult.FIRST_ORDERED_NODE_TYPE, null).singleNodeValue;
                banner.click()
            })()"""
            % xpath
        )
        self.browser.page().runJavaScript(s)


if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    w = MainWindow()
    w.resize(640, 480)
    w.show()
    sys.exit(app.exec_())

Is it possible to click inside a self made web app?

2 Answers2