I'm working on a toy Swift implementation of Teamviewer where users can collaborate. To achieve this, I'm creating a secondary, remotely controlled cursor on macOS using Cocoa. My goal is to allow this secondary cursor to manipulate windows and post mouse events below it. I've managed to create the cursor and successfully made it move and animate within the window.
However, I'm struggling with enabling mouse events to be fired by this secondary cursor. When I try to post synthetic mouse events, it doesn't seem to have any effect. Here's the relevant portion of my code:
func click(at point: CGPoint) {
guard let mouseDown = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
let mouseUp = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) else {
return
}
// Post the events to the session event tap
mouseDown.post(tap: .cgSessionEventTap)
mouseUp.post(tap: .cgSessionEventTap)
}
I visited the following StackOverflow answer but did not get any useful information out of it to help fix the issue. Enabled Accessibility features, tried posting to specific PIDs, tried posting events twice in a row, all to no avail. How to simulate mouse click from Mac App to other Application
Here's the full file if you'd like more context:
//
// AppDelegate.swift
// RemoteCursorDemo
//
// Author: 39050e4eb355e94647722e0580c630798ef4829d on 20/05/23.
//
import Cocoa
import Foundation
class CursorView: NSView {
let image: NSImage
init(image: NSImage) {
self.image = image
super.init(frame: .zero)
}
required init?(coder: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
override func draw(_ dirtyRect: NSRect) {
super.draw(dirtyRect)
image.draw(in: dirtyRect)
}
}
@NSApplicationMain
class AppDelegate: NSObject, NSApplicationDelegate {
var window: NSWindow!
var userCursorView: CursorView?
var remoteCursorView: CursorView?
var timer: Timer?
var destination: CGPoint = .zero
var t: CGFloat = 0
let duration: TimeInterval = 2 // Duration for remote cursor to move between two points
let clickProbability: CGFloat = 0.01 // Probability for the remote cursor to click
func applicationDidFinishLaunching(_ aNotification: Notification) {
let screenRect = NSScreen.main!.frame
window = NSWindow(contentRect: screenRect,
styleMask: .borderless,
backing: .buffered,
defer: false)
window.level = NSWindow.Level(rawValue: Int(CGWindowLevelForKey(.maximumWindow)))
window.backgroundColor = NSColor.clear
window.ignoresMouseEvents = true
let maxHeight: CGFloat = 70.0 // Set your maximum height value here
if let userImage = NSImage(named: "userCursorImage") {
let aspectRatio = userImage.size.width / userImage.size.height
let newWidth = aspectRatio * maxHeight
userCursorView = CursorView(image: userImage)
userCursorView!.frame.size = NSSize(width: newWidth, height: maxHeight)
window.contentView?.addSubview(userCursorView!)
}
if let remoteImage = NSImage(named: "remoteCursorImage") {
let aspectRatio = remoteImage.size.width / remoteImage.size.height
let newWidth = aspectRatio * maxHeight
remoteCursorView = CursorView(image: remoteImage)
remoteCursorView!.frame.size = NSSize(width: newWidth, height: maxHeight)
window.contentView?.addSubview(remoteCursorView!)
// Initialize remote cursor position and destination
remoteCursorView!.frame.origin = randomPointWithinScreen()
destination = randomPointWithinScreen()
}
window.makeKeyAndOrderFront(nil)
window.orderFrontRegardless()
// Hide the system cursor
NSCursor.hide()
// Update cursor position every time the mouse is moved or dragged
NSEvent.addGlobalMonitorForEvents(matching: [.mouseMoved, .leftMouseDragged, .rightMouseDragged]) { [weak self] event in
self?.updateCursorPosition(with: event)
}
// Move the remote cursor every 0.01 second
timer = Timer.scheduledTimer(withTimeInterval: 0.01, repeats: true) { [weak self] _ in
self?.moveRemoteCursor()
}
// Exit the app when pressing the escape key
NSEvent.addGlobalMonitorForEvents(matching: .keyDown) { event in
if event.keyCode == 53 {
NSApplication.shared.terminate(self)
}
}
}
func updateCursorPosition(with event: NSEvent) {
var newLocation = event.locationInWindow
newLocation.y -= userCursorView!.frame.size.height
userCursorView?.frame.origin = newLocation
}
func moveRemoteCursor() {
if remoteCursorView!.frame.origin.distance(to: destination) < 1 || t >= 1 {
destination = randomPointWithinScreen()
t = 0
let windowPoint = remoteCursorView!.frame.origin
let screenPoint = window.convertToScreen(NSRect(origin: windowPoint, size: .zero)).origin
let screenHeight = NSScreen.main?.frame.height ?? 0
let cgScreenPoint = CGPoint(x: screenPoint.x, y: screenHeight - screenPoint.y)
click(at: cgScreenPoint)
} else {
let newPosition = cubicBezier(t: t, start: remoteCursorView!.frame.origin, end: destination)
remoteCursorView?.frame.origin = newPosition
t += CGFloat(0.01 / duration)
}
}
func click(at point: CGPoint) {
guard let mouseDown = CGEvent(mouseEventSource: nil, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left),
let mouseUp = CGEvent(mouseEventSource: nil, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) else {
return
}
// Post the events to the session event tap
mouseDown.post(tap: .cgSessionEventTap)
mouseUp.post(tap: .cgSessionEventTap)
}
func drawClickIndicator() {
let remoteCursorPosition = remoteCursorView!.frame.origin
let box = NSBox(frame: NSRect(x: remoteCursorPosition.x, y: remoteCursorPosition.y, width: 10, height: 10))
box.boxType = .custom
box.borderType = .noBorder
box.fillColor = NSColor.red
box.cornerRadius = 5
box.needsDisplay = true
window.contentView?.addSubview(box)
// Fade out the box after a short delay
DispatchQueue.main.asyncAfter(deadline: .now() + 0.5) {
NSAnimationContext.runAnimationGroup({ context in
context.duration = 0.5
box.animator().alphaValue = 0
}, completionHandler: {
box.removeFromSuperview()
})
}
}
// This function simulates a keyboard event with the specified key code
func keyboardEvent(with keycode: CGKeyCode) {
let keyDown = CGEvent(keyboardEventSource: nil, virtualKey: keycode, keyDown: true)
let keyUp = CGEvent(keyboardEventSource: nil, virtualKey: keycode, keyDown: false)
keyDown?.post(tap: .cghidEventTap)
keyUp?.post(tap: .cghidEventTap)
}
func randomPointWithinScreen() -> CGPoint {
guard let screen = NSScreen.main else { return .zero }
let randomX = CGFloat.random(in: 0...screen.frame.width / 2)
let randomY = CGFloat.random(in: 100...screen.frame.height)
return CGPoint(x: randomX, y: randomY)
}
func cubicBezier(t: CGFloat, start: CGPoint, end: CGPoint) -> CGPoint {
let control1 = CGPoint(x: 2 * start.x / 3 + end.x / 3, y: start.y)
let control2 = CGPoint(x: start.x / 3 + 2 * end.x / 3, y: end.y)
let x = pow(1 - t, 3) * start.x + 3 * pow(1 - t, 2) * t * control1.x + 3 * (1 - t) * pow(t, 2) * control2.x + pow(t, 3) * end.x
let y = pow(1 - t, 3) * start.y + 3 * pow(1 - t, 2) * t * control1.y + 3 * (1 - t) * pow(t, 2) * control2.y + pow(t, 3) * end.y
return CGPoint(x: x, y: y)
}
func applicationWillTerminate(_ aNotification: Notification) {
// Show the system cursor when the application is about to terminate
NSCursor.unhide()
}
}
extension CGPoint {
func distance(to point: CGPoint) -> CGFloat {
return hypot(point.x - x, point.y - y)
}
}
