Avoid numerical instability in resize

Because the layout code rounds down the dimensions of the windows
resizing would often be off by one pixel. The width/height fraction
would not exactly reflect the final computed width and so the resize
code would end up calculating things wrong.

To fix this first snap the container size fractions to the pixel grid
and only then do the resize. Also use round() instead of floor() during
layout to avoid a slightly too small width. This applies in two cases:

1. For the container we are actually resizing using floor() might result
   in being 1px too small.
2. For the other containers it might result in resizing them down by 1px
   and then if the container being resized is the last all those extra
   pixels would make the resize too large.

Fixes #4391

3 files changed, 33 insertions, 6 deletions

diff --git a/include/sway/tree/container.h b/include/sway/tree/container.h
index 099a80897..62c6556bf 100644
--- a/include/sway/tree/container.h
+++ b/include/sway/tree/container.h
@@ -92,6 +92,11 @@ struct sway_container {
 	double width_fraction;
 	double height_fraction;
 
+	// The share of space of the parent container that all children occupy
+	// Used for doing the resize calculations
+	double child_total_width;
+	double child_total_height;
+
 	// These are in layout coordinates.
 	double content_x, content_y;
 	int content_width, content_height;
diff --git a/sway/commands/resize.c b/sway/commands/resize.c
index 0c9af12d1..7ff4ef7b2 100644
--- a/sway/commands/resize.c
+++ b/sway/commands/resize.c
@@ -172,9 +172,19 @@ void container_resize_tiled(struct sway_container *con,
 		if (prev && prev->width - sibling_amount < MIN_SANE_W) {
 			return;
 		}
+		if (con->child_total_width <= 0) {
+			return;
+		}
+
+		// We're going to resize so snap all the width fractions to full pixels
+		// to avoid rounding issues
+		list_t *siblings = container_get_siblings(con);
+		for (int i = 0; i < siblings->length; ++i) {
+			struct sway_container *con = siblings->items[i];
+			con->width_fraction = con->width / con->child_total_width;
+		}
 
-		double amount_fraction =
-			((double)amount / con->width) * con->width_fraction;
+		double amount_fraction = (double)amount / con->child_total_width;
 		double sibling_amount_fraction =
 			prev ? amount_fraction / 2.0 : amount_fraction;
 
@@ -193,9 +203,19 @@ void container_resize_tiled(struct sway_container *con,
 		if (prev && prev->height - sibling_amount < MIN_SANE_H) {
 			return;
 		}
+		if (con->child_total_height <= 0) {
+			return;
+		}
+
+		// We're going to resize so snap all the height fractions to full pixels
+		// to avoid rounding issues
+		list_t *siblings = container_get_siblings(con);
+		for (int i = 0; i < siblings->length; ++i) {
+			struct sway_container *con = siblings->items[i];
+			con->height_fraction = con->height / con->child_total_height;
+		}
 
-		double amount_fraction =
-			((double)amount / con->height) * con->height_fraction;
+		double amount_fraction = (double)amount / con->child_total_height;
 		double sibling_amount_fraction =
 			prev ? amount_fraction / 2.0 : amount_fraction;
 
diff --git a/sway/tree/arrange.c b/sway/tree/arrange.c
index 7b88b3a2d..e4f59110b 100644
--- a/sway/tree/arrange.c
+++ b/sway/tree/arrange.c
@@ -78,9 +78,10 @@ static void apply_horiz_layout(list_t *children, struct wlr_box *parent) {
 	double child_x = parent->x;
 	for (int i = 0; i < children->length; ++i) {
 		struct sway_container *child = children->items[i];
+		child->child_total_width = child_total_width;
 		child->x = child_x;
 		child->y = parent->y;
-		child->width = floor(child->width_fraction * child_total_width);
+		child->width = round(child->width_fraction * child_total_width);
 		child->height = parent->height;
 		child_x += child->width + inner_gap;
 
@@ -156,10 +157,11 @@ static void apply_vert_layout(list_t *children, struct wlr_box *parent) {
 	double child_y = parent->y;
 	for (int i = 0; i < children->length; ++i) {
 		struct sway_container *child = children->items[i];
+		child->child_total_height = child_total_height;
 		child->x = parent->x;
 		child->y = child_y;
 		child->width = parent->width;
-		child->height = floor(child->height_fraction * child_total_height);
+		child->height = round(child->height_fraction * child_total_height);
 		child_y += child->height + inner_gap;
 
 		// Make last child use remaining height of parent