gaps: fix inner gaps for stacked/tabbed containers in splith/splitv (#5275)

Fixes https://github.com/i3/i3/issues/5261

include/con.h

@@ -565,3 +565,9 @@ uint32_t con_rect_size_in_orientation(Con *con);
  *
  */
 void con_merge_into(Con *old, Con *new);
+
+/**
+ * Returns true if the container is within any stacked/tabbed split container.
+ *
+ */
+bool con_inside_stacked_or_tabbed(Con *con);

src/con.c

@@ -2579,3 +2579,18 @@ void con_merge_into(Con *old, Con *new) {
 
     tree_close_internal(old, DONT_KILL_WINDOW, false);
 }
+
+/*
+ * Returns true if the container is within any stacked/tabbed split container.
+ *
+ */
+bool con_inside_stacked_or_tabbed(Con *con) {
+    if (con->parent == NULL) {
+        return false;
+    }
+    if (con->parent->layout == L_STACKED ||
+        con->parent->layout == L_TABBED) {
+        return true;
+    }
+    return con_inside_stacked_or_tabbed(con->parent);
+}

src/gaps.c

@@ -47,20 +47,27 @@ gaps_t calculate_effective_gaps(Con *con) {
  * Decides whether the container should be inset.
  */
 bool gaps_should_inset_con(Con *con, int children) {
+    /* No parent? None of the conditionals below can be true. */
+    if (con->parent == NULL) {
+        return false;
+    }
+
+    const bool leaf_or_stacked_tabbed =
+        con_is_leaf(con) ||
+        (con->layout == L_STACKED || con->layout == L_TABBED);
+
     /* Inset direct children of the workspace that are leaf containers or
        stacked/tabbed containers. */
-    if (con->parent != NULL &&
+    if (leaf_or_stacked_tabbed &&
-        con->parent->type == CT_WORKSPACE &&
+        con->parent->type == CT_WORKSPACE) {
-        (con_is_leaf(con) ||
-         (con->layout == L_STACKED || con->layout == L_TABBED))) {
         return true;
     }
 
     /* Inset direct children of vertical or horizontal split containers at any
-       depth in the tree (only leaf containers, not split containers within
+       depth in the tree. Do not inset as soon as any parent is a stacked or
-       split containers, to avoid double insets). */
+       tabbed container, to avoid double insets. */
-    if (con_is_leaf(con) &&
+    if (leaf_or_stacked_tabbed &&
-        con->parent != NULL &&
+        !con_inside_stacked_or_tabbed(con) &&
         con->parent->type == CT_CON &&
         (con->parent->layout == L_SPLITH ||
          con->parent->layout == L_SPLITV)) {

testcases/t/319-gaps.t

@@ -163,4 +163,43 @@ is_gaps();
 
 exit_gracefully($pid);
 
+################################################################################
+# Ensure stacked/tabbed containers are properly inset even when they are part
+# of a splith/splitv container (issue #5261).
+################################################################################
+
+$config = <<EOT;
+# i3 config file (v4)
+font -misc-fixed-medium-r-normal--13-120-75-75-C-70-iso10646-1
+
+gaps inner 10
+
+default_border pixel 0
+EOT
+
+$pid = launch_with_config($config);
+
+fresh_workspace;
+
+my $helper = open_window;
+$right = open_window;
+sync_with_i3;
+cmd 'focus left';
+cmd 'splitv';
+$left = open_window;
+sync_with_i3;
+cmd 'splith';
+cmd 'layout stacked';
+sync_with_i3;
+$helper->destroy;
+sync_with_i3;
+
+$inner_gaps = 10;
+$outer_gaps = 0;
+$total_gaps = $outer_gaps + $inner_gaps;
+
+is_gaps();
+
+exit_gracefully($pid);
+
 done_testing;