src/output.c: handle outputs with all modes failing

- To reset the mode request we were using wlr_output_state_set_mode(NULL) which is not
  allowed in wlroots. Replace it with removing the mode from the commit manually.

- Before this patch, we would assume that we can't configure an output if it doesn't have
  any fixed mode. This is not true for wayland backed nested outputs. Replace the fixed
  return false with wlr_output_test_state().

- Prevent configuring the output if the test fails as the current behavior was a
  workaround for the hardcoded return false in case there were no fixed modes available.

Fixes: #3428
Reported-By: kode54

src/output.c

@@ -374,8 +374,10 @@ output_test_auto(struct wlr_output *wlr_output, struct wlr_output_state *state,
 	}
 
 	/* Reset mode if none worked (we may still try to commit) */
-	wlr_output_state_set_mode(state, NULL);
-	return false;
+	wlr_log(WLR_DEBUG, "no working fixed mode found for output %s", wlr_output->name);
+	state->committed &= ~WLR_OUTPUT_STATE_MODE;
+
+	return wlr_output_test_state(wlr_output, state);
 }
 
 static void
@@ -390,10 +392,8 @@ configure_new_output(struct server *server, struct output *output)
 			/* is_client_request */ false)) {
 		wlr_log(WLR_INFO, "mode test failed for output %s",
 			wlr_output->name);
-		/*
-		 * Continue anyway. For some reason, the test fails when
-		 * running nested, yet the following commit succeeds.
-		 */
+		wlr_output_state_set_enabled(&output->pending, false);
+		return;
 	}
 
 	if (rc.adaptive_sync == LAB_ADAPTIVE_SYNC_ENABLED) {