Make alpha work on window borders

Up to now, we always asked the X11 server for color allocation ("which
pixel value corresponds to (r,g,b)?", an AllocCollor request).

This commit adds direct support for TrueColor and DirectColor visuals.
In such a visual, the X11 server gives tells us where the red, green,
and blue color components are in a pixel value and we can then just
directly compute the pixel value.

Additionally, this commit adds code that assumes that in a depth=32
visual, the remaining values (after handling red, green, blue) is the
alpha channel for colors. Thus, this adds support for transparent client
borders.

This commit also touches code for the systray. However, the systray must
always use the X11 server's default visual and that one always(?) has
depth=24, i.e. does not support an alpha channel. Thus, the systray
background still cannot be transparent.

Also, in theory this commit should support visuals where some color
component does not have 8 bits, for example RGB565. However, this is
just theoretic and I have no idea how to actually test this (without
jumping through too many hoops).

Fixes: https://github.com/awesomeWM/awesome/issues/162
Signed-off-by: Uli Schlachter <psychon@znc.in>

color.c

@@ -39,15 +39,17 @@
  */
 static bool
 color_parse(const char *colstr, ssize_t len,
-            uint8_t *red, uint8_t *green, uint8_t *blue)
+            uint8_t *red, uint8_t *green, uint8_t *blue, uint8_t *alpha)
 {
     unsigned long colnum;
     char *p;
 
+    *alpha = 0xff;
+
     colnum = strtoul(colstr + 1, &p, 16);
     if(len == 9 && (p - colstr) == 9)
     {
-        /* We ignore the alpha component */
+        *alpha = colnum & 0xff;
         colnum >>= 8;
         len -= 2;
         p -= 2;
@@ -65,19 +67,58 @@ color_parse(const char *colstr, ssize_t len,
     return true;
 }
 
+/** Given a color component value in the range from 0x00 to 0xff and a mask that
+ * specifies where the component is, move the component into place. For example,
+ * component=0x12 and mask=0xff00 return 0x1200. Note that the mask can have a
+ * different width, for example component=0x12 and mask=0xf00 return 0x100.
+ * \param component The intensity of a color component.
+ * \param mask A mask containing a consecutive number of bits set to 1 defining
+ * where the component is.
+ * \return A pixel value containing the giving component in the given component.
+ */
+static uint32_t
+apply_mask(uint8_t component, uint32_t mask)
+{
+    unsigned int shift = 0;
+    unsigned int width = 0;
+
+    // Shift the mask right until the first set bit appears
+    while (mask != 0 && (mask & 0x1) == 0) {
+        shift++;
+        mask >>= 1;
+    }
+    // Shift the mask right until we saw all set bits
+    while ((mask & 0x1) == 1) {
+        width++;
+        mask >>= 1;
+    }
+
+    // The mask consists of [width] set bits followed by [shift] unset bits.
+    // Modify the component accordingly.
+    uint32_t result = component;
+
+    // Scale the result up to the desired width
+    if (width < 8)
+        result >>= (8 - width);
+    if (width > 8)
+        result <<= (width - 8);
+    return result << shift;
+}
+
 /** Send a request to initialize a X color.
  * If you are only interested in the rgba values and don't need the color's
  * pixel value, you should use color_init_unchecked() instead.
  * \param color color_t struct to store color into.
  * \param colstr Color specification.
  * \param len The length of colstr (which still MUST be NULL terminated).
+ * \param visual The visual for which the color is to be allocated.
  * \return request informations.
  */
 color_init_request_t
-color_init_unchecked(color_t *color, const char *colstr, ssize_t len)
+color_init_unchecked(color_t *color, const char *colstr, ssize_t len, xcb_visualtype_t *visual)
 {
     color_init_request_t req;
-    uint8_t red, green, blue;
+    uint8_t red, green, blue, alpha;
 
     p_clear(&req, 1);
 
@@ -90,21 +131,40 @@ color_init_unchecked(color_t *color, const char *colstr, ssize_t len)
     req.color = color;
 
     /* The color is given in RGB value */
-    if(!color_parse(colstr, len, &red, &green, &blue))
+    if(!color_parse(colstr, len, &red, &green, &blue, &alpha))
     {
         warn("awesome: error, invalid color '%s'", colstr);
         req.has_error = true;
         return req;
     }
 
+    req.colstr = colstr;
+    req.has_error = false;
+
+    if (visual->_class == XCB_VISUAL_CLASS_TRUE_COLOR || visual->_class == XCB_VISUAL_CLASS_DIRECT_COLOR) {
+        uint32_t pixel = 0;
+        pixel |= apply_mask(red, visual->red_mask);
+        pixel |= apply_mask(blue, visual->blue_mask);
+        pixel |= apply_mask(green, visual->green_mask);
+        if (draw_visual_depth(globalconf.screen, visual->visual_id) == 32) {
+            /* This is not actually in the X11 protocol, but we assume that this
+             * is an ARGB visual and everything unset in some mask is alpha.
+             */
+            pixel |= apply_mask(alpha, ~(visual->red_mask | visual->blue_mask | visual->green_mask));
+        }
+        req.color->pixel = pixel;
+        req.color->red   = RGB_8TO16(red);
+        req.color->green = RGB_8TO16(green);
+        req.color->blue  = RGB_8TO16(blue);
+        req.color->initialized = true;
+        return req;
+    }
     req.cookie_hexa = xcb_alloc_color_unchecked(globalconf.connection,
                                                 globalconf.default_cmap,
                                                 RGB_8TO16(red),
                                                 RGB_8TO16(green),
                                                 RGB_8TO16(blue));
 
-    req.has_error = false;
-    req.colstr = colstr;
 
     return req;
 }
@@ -118,6 +178,8 @@ color_init_reply(color_init_request_t req)
 {
     if(req.has_error)
         return false;
+    if(req.color->initialized)
+        return true;
 
     xcb_alloc_color_reply_t *hexa_color;
 

color.h

@@ -44,7 +44,7 @@ typedef struct
     const char *colstr;
 } color_init_request_t;
 
-color_init_request_t color_init_unchecked(color_t *, const char *, ssize_t);
+color_init_request_t color_init_unchecked(color_t *, const char *, ssize_t, xcb_visualtype_t *visual);
 bool color_init_reply(color_init_request_t);
 
 int luaA_pushcolor(lua_State *, const color_t);

objects/window.c

@@ -194,7 +194,7 @@ luaA_window_set_border_color(lua_State *L, window_t *window)
     const char *color_name = luaL_checklstring(L, -1, &len);
 
     if(color_name &&
-       color_init_reply(color_init_unchecked(&window->border_color, color_name, len)))
+       color_init_reply(color_init_unchecked(&window->border_color, color_name, len, globalconf.visual)))
     {
         window->border_need_update = true;
         luaA_object_emit_signal(L, -3, "property::border_color", 0);

systray.c

@@ -355,7 +355,7 @@ luaA_systray(lua_State *L)
         color_t bg_color;
         bool force_redraw = false;
 
-        if(color_init_reply(color_init_unchecked(&bg_color, bg, bg_len))
+        if(color_init_reply(color_init_unchecked(&bg_color, bg, bg_len, globalconf.default_visual))
                 && globalconf.systray.background_pixel != bg_color.pixel)
         {
             uint32_t config_back[] = { bg_color.pixel };