common: flesh out enum lab_edge and prefer over wlr_edges/wlr_direction

I like the new common/edge.h. I don't like how inconsistently we use it.

Current situation:

 - enum wlr_edges and wlr_direction are designed to be used as bitset,
   and are defined compatibly

 - enum lab_edge is *also* designed to be used as bitset, but
   incompatible with the others (LEFT/RIGHT come before UP/DOWN)

 - we use an inconsistent mix of all three *AND* uint32_t (usually with
   the WLR_EDGE constants rather than the LAB_EDGE constants), and
   convert between them on an ad-hoc basis, sometimes implicitly

Let's clean this up:

 - reorder enum lab_edge to be compatible with the two wlr enums
   (check this by static_assert)

 - use TOP/BOTTOM naming rather than UP/DOWN (matches wlr_edges)

 - add constants for the remaining possible combinations of the 4 edges

 - use lab_edge for all internal edge/direction fields, consistently

 - add lab_edge_is_cardinal() as a sanity check before casting to
   enum wlr_direction, and then eliminate all of direction.c/h

Instead of "enum wlr_edges direction", we now have
"enum lab_edge direction" which is not that much better. At least we
are now clear that we're overloading one enum with two meanings.

include/common/direction.h

@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef LABWC_DIRECTION_H
-#define LABWC_DIRECTION_H
-
-#include <wlr/types/wlr_output_layout.h>
-#include "common/edge.h"
-
-bool direction_from_edge(enum lab_edge edge, enum wlr_direction *direction);
-enum wlr_direction direction_get_opposite(enum wlr_direction direction);
-
-#endif /* LABWC_DIRECTION_H */

include/common/edge.h

@@ -2,34 +2,77 @@
 #ifndef LABWC_EDGE_H
 #define LABWC_EDGE_H
 
-#include <wayland-server-core.h>
+#include <stdbool.h>
 
 /**
- * Represents an edge or direction (e.g. window tiling, window motion)
+ * Unified/overloaded enum representing edges, corners, and directions.
+ * Used in many different contexts (moving, resizing, tiling) and with
+ * somewhat different semantics depending on context.
+ *
+ * Examples:
+ *   - LAB_EDGE_TOP can also mean "up" or "north".
+ *   - LAB_EDGES_TOP_LEFT can mean "top left corner" or "northwest".
+ *
+ * The enum is designed to be used as a bitset, and combinations of
+ * edges typically mean what you'd expect from the context. For example,
+ * LAB_EDGES_TOP_LEFT is used when resizing a view from its top-left
+ * corner, or when tiling a view in the top-left corner of an output.
+ *
+ * All 16 possible combinations of TOP/BOTTOM/LEFT/RIGHT are listed for
+ * completeness. Not all combinations make sense in all contexts.
+ *
+ * LAB_EDGE_NONE is sometimes used to mean "invalid".
+ *
+ * LAB_EDGE_ANY means "any edge or combination of edges (except NONE)"
+ * and is distinct from LAB_EDGE_ALL (which means all 4 edges).
+ *
+ * LAB_EDGE_TOP/BOTTOM/LEFT/RIGHT match the corresponding values of
+ * enum wlr_edges and enum wlr_direction, so that conversion between
+ * enums can be done with a simple cast.
  */
 enum lab_edge {
-	LAB_EDGE_INVALID = 0,
+	LAB_EDGE_NONE = 0,
 
-	LAB_EDGE_LEFT = (1 << 0),
-	LAB_EDGE_RIGHT = (1 << 1),
-	LAB_EDGE_UP = (1 << 2),
-	LAB_EDGE_DOWN = (1 << 3),
+	LAB_EDGE_TOP = (1 << 0),    /* or UP */
+	LAB_EDGE_BOTTOM = (1 << 1), /* or DOWN */
+	LAB_EDGE_LEFT = (1 << 2),
+	LAB_EDGE_RIGHT = (1 << 3),
 	LAB_EDGE_CENTER = (1 << 4), /* for window tiling */
 	LAB_EDGE_ANY = (1 << 5), /* for window rules */
 
-	/* for window tiling */
-	LAB_EDGE_UPLEFT = (LAB_EDGE_UP | LAB_EDGE_LEFT),
-	LAB_EDGE_UPRIGHT = (LAB_EDGE_UP | LAB_EDGE_RIGHT),
-	LAB_EDGE_DOWNLEFT = (LAB_EDGE_DOWN | LAB_EDGE_LEFT),
-	LAB_EDGE_DOWNRIGHT = (LAB_EDGE_DOWN | LAB_EDGE_RIGHT),
+	/* corners or ordinal directions (NW/NE/SW/SE) */
+	LAB_EDGES_TOP_LEFT = (LAB_EDGE_TOP | LAB_EDGE_LEFT),
+	LAB_EDGES_TOP_RIGHT = (LAB_EDGE_TOP | LAB_EDGE_RIGHT),
+	LAB_EDGES_BOTTOM_LEFT = (LAB_EDGE_BOTTOM | LAB_EDGE_LEFT),
+	LAB_EDGES_BOTTOM_RIGHT = (LAB_EDGE_BOTTOM | LAB_EDGE_RIGHT),
+
+	/* opposite edges */
+	LAB_EDGES_TOP_BOTTOM = (LAB_EDGE_TOP | LAB_EDGE_BOTTOM),
+	LAB_EDGES_LEFT_RIGHT = (LAB_EDGE_LEFT | LAB_EDGE_RIGHT),
+
+	/* all 4 edges */
+	LAB_EDGES_ALL = (LAB_EDGE_TOP | LAB_EDGE_BOTTOM |
+			LAB_EDGE_LEFT | LAB_EDGE_RIGHT),
+
+	/* 3-edge combinations (for completeness) */
+	LAB_EDGES_EXCEPT_TOP = (LAB_EDGES_ALL ^ LAB_EDGE_TOP),
+	LAB_EDGES_EXCEPT_BOTTOM = (LAB_EDGES_ALL ^ LAB_EDGE_BOTTOM),
+	LAB_EDGES_EXCEPT_LEFT = (LAB_EDGES_ALL ^ LAB_EDGE_LEFT),
+	LAB_EDGES_EXCEPT_RIGHT = (LAB_EDGES_ALL ^ LAB_EDGE_RIGHT),
 };
 
 enum lab_edge lab_edge_parse(const char *direction, bool tiled, bool any);
 
+/**
+ * Returns true if edge is TOP, BOTTOM, LEFT, or RIGHT
+ * (i.e. one of the four cardinal directions N/S/W/E)
+ */
+bool lab_edge_is_cardinal(enum lab_edge edge);
+
 /**
  * lab_edge_invert() - select the opposite of a provided edge
  *
- * Returns LAB_EDGE_INVALID for edges other than UP/DOWN/LEFT/RIGHT.
+ * Returns LAB_EDGE_NONE for edges other than TOP/BOTTOM/LEFT/RIGHT.
  *
  * @edge: edge to be inverted
  */

include/edges.h

@@ -4,7 +4,7 @@
 
 #include <limits.h>
 #include <stdbool.h>
-#include <stdint.h>
+#include "common/edge.h"
 #include "common/macros.h"
 
 struct border;
@@ -113,9 +113,10 @@ void edges_adjust_move_coords(struct view *view, struct border edges,
 	int *x, int *y, bool use_pending);
 
 void edges_adjust_resize_geom(struct view *view, struct border edges,
-	uint32_t resize_edges, struct wlr_box *geom, bool use_pending);
+	enum lab_edge resize_edges, struct wlr_box *geom, bool use_pending);
 
 bool edges_traverse_edge(struct edge current, struct edge target, struct edge edge);
 
 void edges_calculate_visibility(struct server *server, struct view *ignored_view);
+
 #endif /* LABWC_EDGES_H */

include/input/cursor.h

@@ -3,7 +3,7 @@
 #define LABWC_CURSOR_H
 
 #include <wlr/types/wlr_cursor.h>
-#include <wlr/util/edges.h>
+#include "common/edge.h"
 #include "ssd.h"
 
 struct view;
@@ -11,7 +11,6 @@ struct seat;
 struct server;
 struct wlr_surface;
 struct wlr_scene_node;
-enum wl_pointer_button_state;
 
 /* Cursors used internally by labwc */
 enum lab_cursors {
@@ -85,20 +84,19 @@ void cursor_set_visible(struct seat *seat, bool visible);
  * This is mostly important when either resizing a window using a
  * keyboard modifier or when using the Resize action from a keybind.
  */
-uint32_t cursor_get_resize_edges(struct wlr_cursor *cursor,
+enum lab_edge cursor_get_resize_edges(struct wlr_cursor *cursor,
 	struct cursor_context *ctx);
 
 /**
- * cursor_get_from_edge - translate wlroots edge enum to lab_cursor enum
- * @resize_edges - WLR_EDGE_ combination like WLR_EDGE_TOP | WLR_EDGE_RIGHT
+ * cursor_get_from_edge - translate lab_edge enum to lab_cursor enum
+ * @resize_edges - edge(s) being resized
  *
- * Returns LAB_CURSOR_DEFAULT on WLR_EDGE_NONE
  * Returns the appropriate lab_cursors enum if @resize_edges
  * is one of the 4 corners or one of the 4 edges.
  *
- * Asserts on invalid edge combinations like WLR_EDGE_LEFT | WLR_EDGE_RIGHT
+ * Returns LAB_CURSOR_DEFAULT on any other value.
  */
-enum lab_cursors cursor_get_from_edge(uint32_t resize_edges);
+enum lab_cursors cursor_get_from_edge(enum lab_edge resize_edges);
 
 /**
  * cursor_update_focus - update cursor focus, may update the cursor icon

include/labwc.h

@@ -209,7 +209,7 @@ struct server {
 	double grab_x, grab_y;
 	/* View geometry when interactive move/resize is requested */
 	struct wlr_box grab_box;
-	uint32_t resize_edges;
+	enum lab_edge resize_edges;
 
 	/*
 	 * 'active_view' is generally the view with keyboard-focus, updated with
@@ -426,7 +426,8 @@ void seat_focus_override_end(struct seat *seat);
  */
 void interactive_anchor_to_cursor(struct server *server, struct wlr_box *geo);
 
-void interactive_begin(struct view *view, enum input_mode mode, uint32_t edges);
+void interactive_begin(struct view *view, enum input_mode mode,
+	enum lab_edge edges);
 void interactive_finish(struct view *view);
 void interactive_cancel(struct view *view);
 
@@ -434,7 +435,7 @@ void interactive_cancel(struct view *view);
  * Returns the edge to snap a window to.
  * For example, if the output-relative cursor position (x,y) fulfills
  * x <= (<snapping><cornerRange>) and y <= (<snapping><range>),
- * then edge1=LAB_EDGE_UP and edge2=LAB_EDGE_LEFT.
+ * then edge1=LAB_EDGE_TOP and edge2=LAB_EDGE_LEFT.
  * The value of (edge1|edge2) can be passed to view_snap_to_edge().
  */
 bool edge_from_cursor(struct seat *seat, struct output **dest_output,

include/snap-constraints.h

@@ -2,21 +2,19 @@
 #ifndef LABWC_SNAP_CONSTRAINTS_H
 #define LABWC_SNAP_CONSTRAINTS_H
 
-#include <wlr/util/edges.h>
-
-#include "common/border.h"
-#include "view.h"
+#include "common/edge.h"
 
+struct view;
 struct wlr_box;
 
-void snap_constraints_set(struct view *view,
-	enum wlr_edges direction, struct wlr_box geom);
+void snap_constraints_set(struct view *view, enum lab_edge direction,
+	struct wlr_box geom);
 
 void snap_constraints_invalidate(struct view *view);
 
 void snap_constraints_update(struct view *view);
 
 struct wlr_box snap_constraints_effective(struct view *view,
-	enum wlr_edges direction, bool use_pending);
+	enum lab_edge direction, bool use_pending);
 
 #endif /* LABWC_SNAP_CONSTRAINTS_H */

include/ssd.h

@@ -2,8 +2,9 @@
 #ifndef LABWC_SSD_H
 #define LABWC_SSD_H
 
-#include <wayland-server-core.h>
 #include "common/border.h"
+#include "common/edge.h"
+#include "config/types.h"
 
 struct wlr_cursor;
 
@@ -101,7 +102,7 @@ struct view *ssd_button_get_view(const struct ssd_button *button);
 /* Public SSD helpers */
 enum ssd_part_type ssd_get_part_type(const struct ssd *ssd,
 	struct wlr_scene_node *node, struct wlr_cursor *cursor);
-uint32_t ssd_resize_edges(enum ssd_part_type type);
+enum lab_edge ssd_resize_edges(enum ssd_part_type type);
 bool ssd_part_contains(enum ssd_part_type whole, enum ssd_part_type candidate);
 enum lab_ssd_mode ssd_mode_parse(const char *mode);
 

include/view.h

@@ -180,7 +180,7 @@ struct view {
 	enum lab_tristate force_tearing;
 	bool visible_on_all_workspaces;
 	enum lab_edge tiled;
-	uint32_t edges_visible;  /* enum wlr_edges bitset */
+	enum lab_edge edges_visible;
 	bool inhibits_keybinds; /* also inhibits mousebinds */
 	xkb_layout_index_t keyboard_layout;