diff --git a/terminal.c b/terminal.c
index 5111cc95..6177f56c 100644
--- a/terminal.c
+++ b/terminal.c
@@ -1313,7 +1313,10 @@ void
 term_mouse_down(struct terminal *term, int button, int row, int col,
                 bool shift, bool alt, bool ctrl)
 {
-    if (term->wl->kbd.shift) {
+    if (term->wl->focused == term &&
+        term->wl->kbd.shift &&
+        !term->wl->kbd.alt && !term->wl->kbd.ctrl && !term->wl->kbd.meta)
+    {
         /* "raw" mouse mode */
         return;
     }
@@ -1350,7 +1353,10 @@ void
 term_mouse_up(struct terminal *term, int button, int row, int col,
               bool shift, bool alt, bool ctrl)
 {
-    if (term->wl->kbd.shift) {
+    if (term->wl->focused == term &&
+        term->wl->kbd.shift &&
+        !term->wl->kbd.alt && !term->wl->kbd.ctrl && !term->wl->kbd.meta)
+    {
         /* "raw" mouse mode */
         return;
     }
@@ -1392,7 +1398,10 @@ void
 term_mouse_motion(struct terminal *term, int button, int row, int col,
                   bool shift, bool alt, bool ctrl)
 {
-    if (term->wl->kbd.shift) {
+    if (term->wl->focused == term &&
+        term->wl->kbd.shift &&
+        !term->wl->kbd.alt && !term->wl->kbd.ctrl && !term->wl->kbd.meta)
+    {
         /* "raw" mouse mode */
         return;
     }