diff --git a/input.c b/input.c
index 1b1a7fff..23a0c3a7 100644
--- a/input.c
+++ b/input.c
@@ -1174,7 +1174,7 @@ wl_pointer_button(void *data, struct wl_pointer *wl_pointer,
 
         switch (state) {
         case WL_POINTER_BUTTON_STATE_PRESSED: {
-            if (button == BTN_LEFT) {
+            if (button == BTN_LEFT && wayl->mouse.count <= 3) {
                 selection_cancel(term);
 
                 switch (wayl->mouse.count) {