selection: selection_on_rows(): use scrollback relative coords

When checking if the current selection intersects with the
region (passed as parameter to the function), use scrollback relative
coordinates.

This fixes an issue where selections crossing the scrollback
wrap-around being misdetected, resulting in either the selection being
canceled while scrolling, even though it wasn’t scrolled out, or the
selection _not_ being canceled, when it _was_ scrolled out.

CHANGELOG.md

@@ -96,6 +96,7 @@
   used ([#1120][1120]).
 * Search matches ending with a double-width character not being
   highlighted correctly.
+* Selection not being cancelled correctly when scrolled out.
 
 [1055]: https://codeberg.org/dnkl/foot/issues/1055
 [1092]: https://codeberg.org/dnkl/foot/issues/1092

selection.c

@@ -64,41 +64,85 @@ selection_get_end(const struct terminal *term)
 bool
 selection_on_rows(const struct terminal *term, int row_start, int row_end)
 {
+    xassert(term->selection.coords.end.row >= 0);
+
     LOG_DBG("on rows: %d-%d, range: %d-%d (offset=%d)",
             term->selection.coords.start.row, term->selection.coords.end.row,
             row_start, row_end, term->grid->offset);
 
-    if (term->selection.coords.end.row < 0)
-        return false;
-
-    xassert(term->selection.coords.start.row != -1);
-
     row_start += term->grid->offset;
     row_end += term->grid->offset;
+    xassert(row_end >= row_start);
 
     const struct coord *start = &term->selection.coords.start;
     const struct coord *end = &term->selection.coords.end;
 
-    if ((row_start <= start->row && row_end >= start->row) ||
-        (row_start <= end->row && row_end >= end->row))
+    const struct grid *grid = term->grid;
+    const int sb_start = grid->offset + term->rows;
+
+    /* Use scrollback relative coords when checking for overlap */
+    const int rel_row_start =
+        grid_row_abs_to_sb_precalc_sb_start(grid, sb_start, row_start);
+    const int rel_row_end =
+        grid_row_abs_to_sb_precalc_sb_start(grid, sb_start, row_start);
+    int rel_sel_start =
+        grid_row_abs_to_sb_precalc_sb_start(grid, sb_start, start->row);
+    int rel_sel_end =
+        grid_row_abs_to_sb_precalc_sb_start(grid, sb_start, end->row);
+
+    if (rel_sel_start > rel_sel_end) {
+        int tmp = rel_sel_start;
+        rel_sel_start = rel_sel_end;
+        rel_sel_end = tmp;
+    }
+
+    if ((rel_row_start <= rel_sel_start && rel_row_end >= rel_sel_start) ||
+        (rel_row_start <= rel_sel_end && rel_row_end >= rel_sel_end))
     {
         /* The range crosses one of the selection boundaries */
         return true;
     }
 
-    /* For the last check we must ensure start <= end */
-    if (start->row > end->row) {
-        const struct coord *tmp = start;
-        start = end;
-        end = tmp;
-    }
-
-    if (row_start >= start->row && row_end <= end->row)
+    if (rel_row_start >= rel_sel_start && rel_row_end <= rel_sel_end)
         return true;
 
     return false;
 }
 
+void
+selection_scroll_up(struct terminal *term, int rows)
+{
+    xassert(term->selection.coords.end.row >= 0);
+
+    const int rel_row_start =
+        grid_row_abs_to_sb(term->grid, term->rows, term->selection.coords.start.row);
+    const int rel_row_end =
+        grid_row_abs_to_sb(term->grid, term->rows, term->selection.coords.end.row);
+    const int actual_start = min(rel_row_start, rel_row_end);
+
+    if (actual_start - rows < 0) {
+        /* Part of the selection will be scrolled out, cancel it */
+        selection_cancel(term);
+    }
+}
+
+void
+selection_scroll_down(struct terminal *term, int rows)
+{
+    xassert(term->selection.coords.end.row >= 0);
+
+    const int rel_row_start =
+        grid_row_abs_to_sb(term->grid, term->rows, term->selection.coords.start.row);
+    const int rel_row_end =
+        grid_row_abs_to_sb(term->grid, term->rows, term->selection.coords.end.row);
+    const int actual_end = max(rel_row_start, rel_row_end);
+
+    if (actual_end + rows <= term->grid->num_rows) {
+        /* Part of the selection will be scrolled out, cancel it */
+        selection_cancel(term);
+    }
+}
+
 void
 selection_view_up(struct terminal *term, int new_view)
 {

selection.h

@@ -22,6 +22,8 @@ void selection_extend(
 
 bool selection_on_rows(const struct terminal *term, int start, int end);
 
+void selection_scroll_up(struct terminal *term, int rows);
+void selection_scroll_down(struct terminal *term, int rows);
 void selection_view_up(struct terminal *term, int new_view);
 void selection_view_down(struct terminal *term, int new_view);
 

terminal.c

@@ -2537,11 +2537,11 @@ term_scroll_partial(struct terminal *term, struct scroll_region region, int rows
          * scrolled in (i.e. re-used lines).
          */
         if (selection_on_top_region(term, region) ||
-            selection_on_bottom_region(term, region) ||
-            selection_on_rows(term, region.end - rows, region.end - 1))
+            selection_on_bottom_region(term, region))
         {
             selection_cancel(term);
-        }
+        } else
+            selection_scroll_up(term, rows);
     }
 
     sixel_scroll_up(term, rows);
@@ -2611,11 +2611,11 @@ term_scroll_reverse_partial(struct terminal *term,
          * scrolled in (i.e. re-used lines).
          */
         if (selection_on_top_region(term, region) ||
-            selection_on_bottom_region(term, region) ||
-            selection_on_rows(term, region.start, region.start + rows - 1))
+            selection_on_bottom_region(term, region))
         {
             selection_cancel(term);
-        }
+        } else
+            selection_scroll_down(term, rows);
     }
 
     sixel_scroll_down(term, rows);