[v2,1/2] ui/cocoa: capture all keys and combos when mouse is grabbed

Commit Message

Akihiko Odaki March 6, 2022, 12:11 p.m. UTC

From: Gustavo Noronha Silva <[e-mail]>

Applications such as Gnome may use Alt-Tab and Super-Tab for different
purposes, some use Ctrl-arrows so we want to allow qemu to handle
everything when it captures the mouse/keyboard.

However, Mac OS handles some combos like Command-Tab and Ctrl-arrows
at an earlier part of the event handling chain, not letting qemu see it.

We add a global Event Tap that allows qemu to see all events when the
mouse is grabbed. Note that this requires additional permissions.

See:

https://developer.apple.com/documentation/coregraphics/1454426-cgeventtapcreate?language=objc#discussion
https://support.apple.com/en-in/guide/mac-help/mh32356/mac

Acked-by: Markus Armbruster <[e-mail]>
Signed-off-by: Gustavo Noronha Silva <[e-mail]>
Message-Id: <[e-mail]>
Signed-off-by: Akihiko Odaki <[e-mail]>
---
 qapi/ui.json    | 16 ++++++++++++
 qemu-options.hx |  3 +++
 ui/cocoa.m      | 65 ++++++++++++++++++++++++++++++++++++++++++++++++-
 3 files changed, 83 insertions(+), 1 deletion(-)

Comments

BALATON Zoltan March 6, 2022, 2:23 p.m. UTC | #1

On Sun, 6 Mar 2022, Akihiko Odaki wrote:
> From: Gustavo Noronha Silva <[e-mail]>
>
> Applications such as Gnome may use Alt-Tab and Super-Tab for different
> purposes, some use Ctrl-arrows so we want to allow qemu to handle
> everything when it captures the mouse/keyboard.
>
> However, Mac OS handles some combos like Command-Tab and Ctrl-arrows
> at an earlier part of the event handling chain, not letting qemu see it.
>
> We add a global Event Tap that allows qemu to see all events when the
> mouse is grabbed. Note that this requires additional permissions.
>
> See:
>
> https://developer.apple.com/documentation/coregraphics/1454426-cgeventtapcreate?language=objc#discussion
> https://support.apple.com/en-in/guide/mac-help/mh32356/mac
>
> Acked-by: Markus Armbruster <[e-mail]>
> Signed-off-by: Gustavo Noronha Silva <[e-mail]>
> Message-Id: <[e-mail]>
> Signed-off-by: Akihiko Odaki <[e-mail]>
> ---
> qapi/ui.json    | 16 ++++++++++++
> qemu-options.hx |  3 +++
> ui/cocoa.m      | 65 ++++++++++++++++++++++++++++++++++++++++++++++++-
> 3 files changed, 83 insertions(+), 1 deletion(-)
>
> diff --git a/qapi/ui.json b/qapi/ui.json
> index 4a13f883a30..ff0a04da792 100644
> --- a/qapi/ui.json
> +++ b/qapi/ui.json
> @@ -1260,6 +1260,21 @@
> { 'struct'  : 'DisplayCurses',
>   'data'    : { '*charset'       : 'str' } }
>
> +##
> +# @DisplayCocoa:
> +#
> +# Cocoa display options.
> +#
> +# @full-grab: Capture all key presses, including system combos. This
> +#             requires accessibility permissions, since it performs
> +#             a global grab on key events. (default: off)
> +#             See https://support.apple.com/en-in/guide/mac-help/mh32356/mac
> +#
> +# Since: 7.0
> +##
> +{ 'struct'  : 'DisplayCocoa',
> +  'data'    : { '*full-grab'     : 'bool' } }
> +
> ##
> # @DisplayType:
> #
> @@ -1338,6 +1353,7 @@
>   'discriminator' : 'type',
>   'data'    : {
>       'gtk': { 'type': 'DisplayGTK', 'if': 'CONFIG_GTK' },
> +      'cocoa': { 'type': 'DisplayCocoa', 'if': 'CONFIG_COCOA' },
>       'curses': { 'type': 'DisplayCurses', 'if': 'CONFIG_CURSES' },
>       'egl-headless': { 'type': 'DisplayEGLHeadless',
>                         'if': { 'all': ['CONFIG_OPENGL', 'CONFIG_GBM'] } },
> diff --git a/qemu-options.hx b/qemu-options.hx
> index 094a6c1d7c2..4df9ccc3446 100644
> --- a/qemu-options.hx
> +++ b/qemu-options.hx
> @@ -1916,6 +1916,9 @@ DEF("display", HAS_ARG, QEMU_OPTION_display,
> #if defined(CONFIG_CURSES)
>     "-display curses[,charset=<encoding>]\n"
> #endif
> +#if defined(CONFIG_COCOA)
> +    "-display cocoa[,full_grab=on|off]\n"
> +#endif
> #if defined(CONFIG_OPENGL)
>     "-display egl-headless[,rendernode=<file>]\n"
> #endif
> diff --git a/ui/cocoa.m b/ui/cocoa.m
> index 8ab9ab5e84d..bfd602a96b9 100644
> --- a/ui/cocoa.m
> +++ b/ui/cocoa.m
> @@ -308,11 +308,13 @@ @interface QemuCocoaView : NSView
>     BOOL isMouseGrabbed;
>     BOOL isFullscreen;
>     BOOL isAbsoluteEnabled;
> +    CFMachPortRef eventsTap;
> }
> - (void) switchSurface:(pixman_image_t *)image;
> - (void) grabMouse;
> - (void) ungrabMouse;
> - (void) toggleFullScreen:(id)sender;
> +- (void) setFullGrab:(id)sender;
> - (void) handleMonitorInput:(NSEvent *)event;
> - (bool) handleEvent:(NSEvent *)event;
> - (bool) handleEventLocked:(NSEvent *)event;
> @@ -335,6 +337,19 @@ - (void) raiseAllKeys;
>
> QemuCocoaView *cocoaView;
>
> +static CGEventRef handleTapEvent(CGEventTapProxy proxy, CGEventType type, CGEventRef cgEvent, void *userInfo)
> +{
> +    QemuCocoaView *cocoaView = userInfo;
> +    NSEvent* event = [NSEvent eventWithCGEvent:cgEvent];

Sorry, I've missed this before but the space in NSEvent* event is also on 
the wrong side of the * it should be NSEvent *event. Does checkpatch miss 
these or it can't handle Objective-C? Another one below...

> +    if ([cocoaView isMouseGrabbed] && [cocoaView handleEvent:event]) {
> +        COCOA_DEBUG("Global events tap: qemu handled the event, capturing!\n");
> +        return NULL;
> +    }
> +    COCOA_DEBUG("Global events tap: qemu did not handle the event, letting it through...\n");
> +
> +    return cgEvent;
> +}
> +
> @implementation QemuCocoaView
> - (id)initWithFrame:(NSRect)frameRect
> {
> @@ -360,6 +375,11 @@ - (void) dealloc
>     }
>
>     qkbd_state_free(kbd);
> +
> +    if (eventsTap) {
> +        CFRelease(eventsTap);
> +    }
> +
>     [super dealloc];
> }
>
> @@ -654,6 +674,36 @@ - (void) toggleFullScreen:(id)sender
>     }
> }
>
> +- (void) setFullGrab:(id)sender
> +{
> +    COCOA_DEBUG("QemuCocoaView: setFullGrab\n");
> +
> +    CGEventMask mask = CGEventMaskBit(kCGEventKeyDown) | CGEventMaskBit(kCGEventKeyUp) | CGEventMaskBit(kCGEventFlagsChanged);
> +    eventsTap = CGEventTapCreate(kCGHIDEventTap, kCGHeadInsertEventTap, kCGEventTapOptionDefault,
> +                                 mask, handleTapEvent, self);
> +    if (!eventsTap) {
> +        warn_report("Could not create event tap, system key combos will not be captured.\n");
> +        return;
> +    } else {
> +        COCOA_DEBUG("Global events tap created! Will capture system key combos.\n");
> +    }
> +
> +    CFRunLoopRef runLoop = CFRunLoopGetCurrent();
> +    if (!runLoop) {
> +        warn_report("Could not obtain current CF RunLoop, system key combos will not be captured.\n");
> +        return;
> +    }
> +
> +    CFRunLoopSourceRef tapEventsSrc = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, eventsTap, 0);
> +    if (!tapEventsSrc ) {
> +        warn_report("Could not obtain current CF RunLoop, system key combos will not be captured.\n");
> +        return;
> +    }
> +
> +    CFRunLoopAddSource(runLoop, tapEventsSrc, kCFRunLoopDefaultMode);
> +    CFRelease(tapEventsSrc);
> +}
> +
> - (void) toggleKey: (int)keycode {
>     qkbd_state_key_event(kbd, keycode, !qkbd_state_key_get(kbd, keycode));
> }
> @@ -1281,6 +1331,13 @@ - (void)toggleFullScreen:(id)sender
>     [cocoaView toggleFullScreen:sender];
> }
>
> +- (void) setFullGrab:(id)sender
> +{
> +    COCOA_DEBUG("QemuCocoaAppController: setFullGrab\n");
> +
> +    [cocoaView setFullGrab:sender];
> +}
> +
> /* Tries to find then open the specified filename */
> - (void) openDocumentation: (NSString *) filename
> {
> @@ -2057,11 +2114,17 @@ static void cocoa_display_init(DisplayState *ds, DisplayOptions *opts)
>     qemu_sem_wait(&app_started_sem);
>     COCOA_DEBUG("cocoa_display_init: app start completed\n");
>
> +    QemuCocoaAppController* controller = (QemuCocoaAppController*)[[NSApplication sharedApplication] delegate];

Both in the type declaration and in the cast on the right hand side space 
is missing between type and *.

Otherwise I'd give an R-b but it does not worth much as I've never used 
CGEventTap and have not checked the docs so maybe someone else who 
actually knows this could better review it.

Regards,
BALATON Zoltan

>     /* if fullscreen mode is to be used */
>     if (opts->has_full_screen && opts->full_screen) {
>         dispatch_async(dispatch_get_main_queue(), ^{
>             [NSApp activateIgnoringOtherApps: YES];
> -            [(QemuCocoaAppController *)[[NSApplication sharedApplication] delegate] toggleFullScreen: nil];
> +            [controller toggleFullScreen: nil];
> +        });
> +    }
> +    if (opts->u.cocoa.has_full_grab && opts->u.cocoa.full_grab) {
> +        dispatch_async(dispatch_get_main_queue(), ^{
> +            [controller setFullGrab: nil];
>         });
>     }
>     if (opts->has_show_cursor && opts->show_cursor) {
>

Patch

diff --git a/qapi/ui.json b/qapi/ui.json
index 4a13f883a30..ff0a04da792 100644
--- a/qapi/ui.json
+++ b/qapi/ui.json
@@ -1260,6 +1260,21 @@ 
 { 'struct'  : 'DisplayCurses',
   'data'    : { '*charset'       : 'str' } }
 
+##
+# @DisplayCocoa:
+#
+# Cocoa display options.
+#
+# @full-grab: Capture all key presses, including system combos. This
+#             requires accessibility permissions, since it performs
+#             a global grab on key events. (default: off)
+#             See https://support.apple.com/en-in/guide/mac-help/mh32356/mac
+#
+# Since: 7.0
+##
+{ 'struct'  : 'DisplayCocoa',
+  'data'    : { '*full-grab'     : 'bool' } }
+
 ##
 # @DisplayType:
 #
@@ -1338,6 +1353,7 @@ 
   'discriminator' : 'type',
   'data'    : {
       'gtk': { 'type': 'DisplayGTK', 'if': 'CONFIG_GTK' },
+      'cocoa': { 'type': 'DisplayCocoa', 'if': 'CONFIG_COCOA' },
       'curses': { 'type': 'DisplayCurses', 'if': 'CONFIG_CURSES' },
       'egl-headless': { 'type': 'DisplayEGLHeadless',
                         'if': { 'all': ['CONFIG_OPENGL', 'CONFIG_GBM'] } },
diff --git a/qemu-options.hx b/qemu-options.hx
index 094a6c1d7c2..4df9ccc3446 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -1916,6 +1916,9 @@  DEF("display", HAS_ARG, QEMU_OPTION_display,
 #if defined(CONFIG_CURSES)
     "-display curses[,charset=<encoding>]\n"
 #endif
+#if defined(CONFIG_COCOA)
+    "-display cocoa[,full_grab=on|off]\n"
+#endif
 #if defined(CONFIG_OPENGL)
     "-display egl-headless[,rendernode=<file>]\n"
 #endif
diff --git a/ui/cocoa.m b/ui/cocoa.m
index 8ab9ab5e84d..bfd602a96b9 100644
--- a/ui/cocoa.m
+++ b/ui/cocoa.m
@@ -308,11 +308,13 @@  @interface QemuCocoaView : NSView
     BOOL isMouseGrabbed;
     BOOL isFullscreen;
     BOOL isAbsoluteEnabled;
+    CFMachPortRef eventsTap;
 }
 - (void) switchSurface:(pixman_image_t *)image;
 - (void) grabMouse;
 - (void) ungrabMouse;
 - (void) toggleFullScreen:(id)sender;
+- (void) setFullGrab:(id)sender;
 - (void) handleMonitorInput:(NSEvent *)event;
 - (bool) handleEvent:(NSEvent *)event;
 - (bool) handleEventLocked:(NSEvent *)event;
@@ -335,6 +337,19 @@  - (void) raiseAllKeys;
 
 QemuCocoaView *cocoaView;
 
+static CGEventRef handleTapEvent(CGEventTapProxy proxy, CGEventType type, CGEventRef cgEvent, void *userInfo)
+{
+    QemuCocoaView *cocoaView = userInfo;
+    NSEvent* event = [NSEvent eventWithCGEvent:cgEvent];
+    if ([cocoaView isMouseGrabbed] && [cocoaView handleEvent:event]) {
+        COCOA_DEBUG("Global events tap: qemu handled the event, capturing!\n");
+        return NULL;
+    }
+    COCOA_DEBUG("Global events tap: qemu did not handle the event, letting it through...\n");
+
+    return cgEvent;
+}
+
 @implementation QemuCocoaView
 - (id)initWithFrame:(NSRect)frameRect
 {
@@ -360,6 +375,11 @@  - (void) dealloc
     }
 
     qkbd_state_free(kbd);
+
+    if (eventsTap) {
+        CFRelease(eventsTap);
+    }
+
     [super dealloc];
 }
 
@@ -654,6 +674,36 @@  - (void) toggleFullScreen:(id)sender
     }
 }
 
+- (void) setFullGrab:(id)sender
+{
+    COCOA_DEBUG("QemuCocoaView: setFullGrab\n");
+
+    CGEventMask mask = CGEventMaskBit(kCGEventKeyDown) | CGEventMaskBit(kCGEventKeyUp) | CGEventMaskBit(kCGEventFlagsChanged);
+    eventsTap = CGEventTapCreate(kCGHIDEventTap, kCGHeadInsertEventTap, kCGEventTapOptionDefault,
+                                 mask, handleTapEvent, self);
+    if (!eventsTap) {
+        warn_report("Could not create event tap, system key combos will not be captured.\n");
+        return;
+    } else {
+        COCOA_DEBUG("Global events tap created! Will capture system key combos.\n");
+    }
+
+    CFRunLoopRef runLoop = CFRunLoopGetCurrent();
+    if (!runLoop) {
+        warn_report("Could not obtain current CF RunLoop, system key combos will not be captured.\n");
+        return;
+    }
+
+    CFRunLoopSourceRef tapEventsSrc = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, eventsTap, 0);
+    if (!tapEventsSrc ) {
+        warn_report("Could not obtain current CF RunLoop, system key combos will not be captured.\n");
+        return;
+    }
+
+    CFRunLoopAddSource(runLoop, tapEventsSrc, kCFRunLoopDefaultMode);
+    CFRelease(tapEventsSrc);
+}
+
 - (void) toggleKey: (int)keycode {
     qkbd_state_key_event(kbd, keycode, !qkbd_state_key_get(kbd, keycode));
 }
@@ -1281,6 +1331,13 @@  - (void)toggleFullScreen:(id)sender
     [cocoaView toggleFullScreen:sender];
 }
 
+- (void) setFullGrab:(id)sender
+{
+    COCOA_DEBUG("QemuCocoaAppController: setFullGrab\n");
+
+    [cocoaView setFullGrab:sender];
+}
+
 /* Tries to find then open the specified filename */
 - (void) openDocumentation: (NSString *) filename
 {
@@ -2057,11 +2114,17 @@  static void cocoa_display_init(DisplayState *ds, DisplayOptions *opts)
     qemu_sem_wait(&app_started_sem);
     COCOA_DEBUG("cocoa_display_init: app start completed\n");
 
+    QemuCocoaAppController* controller = (QemuCocoaAppController*)[[NSApplication sharedApplication] delegate];
     /* if fullscreen mode is to be used */
     if (opts->has_full_screen && opts->full_screen) {
         dispatch_async(dispatch_get_main_queue(), ^{
             [NSApp activateIgnoringOtherApps: YES];
-            [(QemuCocoaAppController *)[[NSApplication sharedApplication] delegate] toggleFullScreen: nil];
+            [controller toggleFullScreen: nil];
+        });
+    }
+    if (opts->u.cocoa.has_full_grab && opts->u.cocoa.full_grab) {
+        dispatch_async(dispatch_get_main_queue(), ^{
+            [controller setFullGrab: nil];
         });
     }
     if (opts->has_show_cursor && opts->show_cursor) {