Refuse attempts to resize clients to invalid size

X11 does not allow to resize a window to size 0x0. Also, there are some
possibilities of integer overflows in our case. We tried to handle this
already, but there was a loop-hole: If the too-small-value is only
produced after applying size hints, then this was not caught.

Fix this by applying size hints before checking if the resulting size is
valid. However, this means some check needs to be duplicated to handle
the possibility of integer underflows while applying size hints.

Helps-with: https://github.com/awesomeWM/awesome/issues/1340
Signed-off-by: Uli Schlachter <psychon@znc.in>

objects/client.c

@@ -1678,6 +1678,17 @@ client_resize(client_t *c, area_t geometry, bool honor_hints)
     if(geometry.y + geometry.height < 0)
         geometry.y = 0;
 
+    if (honor_hints) {
+        /* We could get integer underflows in client_remove_titlebar_geometry()
+         * without these checks here.
+         */
+        if(geometry.width < c->titlebar[CLIENT_TITLEBAR_LEFT].size + c->titlebar[CLIENT_TITLEBAR_RIGHT].size)
+            return false;
+        if(geometry.height < c->titlebar[CLIENT_TITLEBAR_TOP].size + c->titlebar[CLIENT_TITLEBAR_BOTTOM].size)
+            return false;
+        geometry = client_apply_size_hints(c, geometry);
+    }
+
     if(geometry.width < c->titlebar[CLIENT_TITLEBAR_LEFT].size + c->titlebar[CLIENT_TITLEBAR_RIGHT].size)
         return false;
     if(geometry.height < c->titlebar[CLIENT_TITLEBAR_TOP].size + c->titlebar[CLIENT_TITLEBAR_BOTTOM].size)
@@ -1686,9 +1697,6 @@ client_resize(client_t *c, area_t geometry, bool honor_hints)
     if(geometry.width == 0 || geometry.height == 0)
         return false;
 
-    if (honor_hints)
-        geometry = client_apply_size_hints(c, geometry);
-
     if(!AREA_EQUAL(c->geometry, geometry))
     {
         client_resize_do(c, geometry);

tests/test-resize.lua

@@ -439,6 +439,37 @@ table.insert(steps, function()
     return true
 end)
 
+table.insert(steps, function()
+    for _, c in pairs(client.get()) do
+        c:kill()
+    end
+    if #client.get() == 0 then
+        test_client(nil, nil, nil, nil, true)
+        return true
+    end
+end)
+
+table.insert(steps, function()
+    if #client.get() ~= 1 then
+        return
+    end
+
+    local c = client.get()[1]
+    local geo = c:geometry()
+    local hints = c.size_hints
+    assert(hints.height_inc == 200)
+    assert(hints.width_inc == 200)
+    assert(c:apply_size_hints(1, 1) == 0)
+
+    c:geometry { width = 1, height = 50 }
+
+    -- The above should be rejected, because it would make us resize the
+    -- window size 0x0.
+    assert(c:geometry().width == geo.width)
+    assert(c:geometry().height == geo.height)
+    return true
+end)
+
 require("_runner").run_steps(steps)
 
 -- vim: filetype=lua:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80