ci: overhaul coverage processing

- Execute the tests without compiling, and don't mess with the source
   files when coverage is enabled.
   This ensures that the coverage report lines are correct.
   This disables the doc tests, as their results would be unused.
   Hack: it still expands macros on util.lua, because of
   `util.get_themes_dir()` and `util.get_awesome_icon_dir()`, which might
   be moved later.  Ref:
   https://github.com/awesomeWM/awesome/pull/1239#discussion_r93828178.
 - ensure that BUILD_APIDOC and DO_COVERAGE are not used together
 - awesomeConfig.cmake: add DO_COVERAGE as an option
 - Travis: only install codecov with DO_COVERAGE=codecov
 - Travis: do not use set -v; use set -x with DO_COVERAGE
 - do not use trailing slashes with dirs in tests/examples/CMakeLists.txt / .luacov
 - Use latest luacov (0.12.0-1) again
   This reverts parts of 4cc6a815.
   I think it is better to fix any failure that 4cc6a815 tried to work around.
 - Travis: simplify/fix require('luacov') for functionaltests
 - tests/examples/CMakeLists.txt: resolve ../.. in SOURCE_DIR
 - tests/examples/CMakeLists.txt: add DO_COVERAGE to env
 - Cleanup/simplify .luacov: work with SOURCE_DIRECTORY alone
 - tests/run.sh: pass through / set SOURCE_DIRECTORY when running awesome
 - tests/run.sh: resolve source_dir
 - use DO_COVERAGE from env only

.luacov

@@ -1,33 +1,29 @@
 -- Configuration file for LuaCov
 
 -- This variable is set externally
-local build = os.getenv("BUILD_DIRECTORY") or error("$BUILD_DIRECTORY not set")
-local lib_dir = os.getenv("AWESOME_LIB_DIR") or build .. "lib/"
-local source = os.getenv("SOURCE_DIRECTORY")
+local source = os.getenv("SOURCE_DIRECTORY") or os.getenv('PWD')
+local build = os.getenv("BUILD_DIRECTORY") or (source .. "/build")
+local lib_dir = os.getenv("AWESOME_LIB_DIR") or (source .. "/lib")
 
 local function escape_pattern(str)
     return string.gsub(str, "%W", "%%%1")
 end
 
 return {
-    statsfile = build .. "luacov.stats.out",
+    statsfile = build .. "/luacov.stats.out",
 
     include = {
-        escape_pattern(lib_dir) .. ".+",
+        escape_pattern(lib_dir) .. "/.+",
 
         -- For things already having the correct path
         -- (happens with integration tests)
         "^lib/",
-
-        -- For the shape-API auto-generated images
-        source and escape_pattern(source .. "lib/"),
     },
 
     -- configuration for luacov-coveralls reporter
     coveralls = {
         pathcorrect = {
-            { escape_pattern(lib_dir), "lib/"},
-            source and { escape_pattern(source .. "lib/"), "lib/" },
+            { escape_pattern(lib_dir..'/'), "lib/"},
         },
     },
 }

.travis.yml

@@ -24,6 +24,7 @@ env:
     - secure: "R/HYDclnws1I1+v9Yjt+RKa4CsFhbBT9tiwE3EfPhEj2KCYX4sFRMxuZvLf5sq0XWdrQaPhQ54fgAZGr3f054JKRXcTB0g9J6nhSHz9kIjPh446gafUhEeDQcZRwM/MeCWiwFIkiZm6smYoDFE9JTWu6quNV+lQ4kcVDOp2ibEc="
 
 before_install:
+  - if [ "$BUILD_APIDOC" = true ] && [ -n "$DO_COVERAGE" ]; then echo "BUILD_APIDOC and DO_COVERAGE are not meant to be used together." >&2; exit 1; fi
   - if [ -z $LUAINCLUDE ]; then LUAINCLUDE=/usr/include/${LUANAME}; fi
   - if [ -z $LUALIBRARY ]; then LUALIBRARY=/usr/lib/x86_64-linux-gnu/lib${LUANAME}.so; fi
   - cmake --version
@@ -49,7 +50,7 @@ install:
   # Install Lua (per env).
   # Note that Lua 5.3 is installed manually, because it is not available in Ubuntu Trusty.
   - |
-    set -ev
+    set -e
     if [[ "$LUA" == "5.3" ]]; then
       wget http://www.lua.org/ftp/lua-5.3.3.tar.gz -O lua.tar.gz
       tar -xvzf lua.tar.gz
@@ -94,17 +95,19 @@ install:
 
   # Install dependencies for code coverage testing.
   - if [ "$DO_COVERAGE" = "" ]; then export DO_COVERAGE=0; fi
-  - if [ "$DO_COVERAGE" != "0" ]; then sudo luarocks install luacov 0.11.0-1; fi
+  - if [ "$DO_COVERAGE" != "0" ]; then sudo luarocks install luacov 0.12.0-1; fi
   - if [ "$DO_COVERAGE" = "coveralls" ]; then sudo luarocks install luacov-coveralls; fi
 
   # Determine custom version.
   - export AWESOME_VERSION="${TRAVIS_BRANCH}-g$(git rev-parse --short HEAD)"
   - 'if [ "$TRAVIS_PULL_REQUEST" != false ]; then AWESOME_VERSION="${AWESOME_VERSION}-PR${TRAVIS_PULL_REQUEST}"; fi'
   # function for codecov support
-  - travis_retry wget -O /tmp/codecov-bash https://codecov.io/bash
+  - if [ "$DO_COVERAGE" = "codecov" ]; then travis_retry wget -O /tmp/codecov-bash https://codecov.io/bash; fi
   - |
     do_codecov() {
+      echo "== do_codecov in $PWD: $*: build/luacov.stats.out: =="
       if [ "$DO_COVERAGE" = "codecov" ]; then
+        cat build/luacov.stats.out
         (cd build && luacov) || return 1
         travis_retry bash /tmp/codecov-bash -X coveragepy -c -F "$1" || return 1
         rm build/luacov.stats.out
@@ -130,9 +133,9 @@ install:
       return $result
     }
 script:
-  - export CMAKE_ARGS="-DLUA_LIBRARY=${LUALIBRARY} -DLUA_INCLUDE_DIR=${LUAINCLUDE} -D OVERRIDE_VERSION=$AWESOME_VERSION -D DO_COVERAGE=${DO_COVERAGE} -DSTRICT_TESTS=true"
+  - export CMAKE_ARGS="-DLUA_LIBRARY=${LUALIBRARY} -DLUA_INCLUDE_DIR=${LUAINCLUDE} -D OVERRIDE_VERSION=$AWESOME_VERSION -DSTRICT_TESTS=true"
   - |
-    set -ev
+    set -e
     if [ -n "$BUILD_IN_DIR" ]; then
       # Explicitly remove the Makefile to not build from the src dir accidentally.
       rm Makefile
@@ -147,20 +150,23 @@ script:
       travis_run_in_fold "make.install" sudo env PATH=$PATH make install
       awesome --version
     fi
-  - |
-    if [ "$TRAVIS_TEST_RESULT" = 0 ]; then
-      do_codecov samples
-    fi
   # Run check-unit{,-coverage} and check-integration.
   - |
-    set -ev
+    set -ex
     if [ "$TRAVIS_TEST_RESULT" = 0 ]; then
       if [ "$DO_COVERAGE" != "0" ]; then
         travis_fold_start "DO_COVERAGE"
+
+        # Run tests/examples explicitly.
+        (mkdir -p tests/examples/build \
+         && cd tests/examples/build \
+         && cmake $CMAKE_ARGS ..)
+        do_codecov samples
+
         (make check-unit-coverage \
          && do_codecov unittests \
-         && sed -i "1 i\\require('luacov.runner')('"$PWD"/.luacov')" build/awesomerc.lua \
-         && BUILD_DIRECTORY="" tests/run.sh \
+         && sed -i "1 irequire('luacov.runner')('$PWD/.luacov')" build/awesomerc.lua \
+         && tests/run.sh \
          && do_codecov functionaltests)
         ret=$?
         travis_fold_end
@@ -176,7 +182,7 @@ script:
       travis_run_in_fold "make.check-qa" make check-qa
     fi
   - |
-    set -ev
+    set -e
     if [ "$TEST_PREV_COMMITS" = 1 ] && ! [ "$TRAVIS_PULL_REQUEST" = false ]; then
       # Check each commit separately (to make git-bisect less annoying).
       # Fix Travis' commit range (https://github.com/travis-ci/travis-ci/issues/4596).
@@ -210,4 +216,9 @@ after_success:
   # Push updated API docs for relevant branches, e.g. non-PRs builds on master.
   - if [ "$BUILD_APIDOC" = "true" ]; then build-utils/travis-apidoc.sh; fi
   # Push code coverage information
-  - if [ "$DO_COVERAGE" = "coveralls" ]; then BUILD_DIRECTORY="$(pwd)/$(readlink build)/" SOURCE_DIRECTORY="$(pwd)/" luacov-coveralls --verbose --merge; fi
+  - |
+    set -e
+    if [ "$DO_COVERAGE" = "coveralls" ]; then
+      cat build/luacov.stats.out
+      luacov-coveralls --verbose --merge
+    fi

awesomeConfig.cmake

@@ -12,6 +12,9 @@ option(WITH_DBUS "build with D-BUS" ON)
 option(GENERATE_MANPAGES "generate manpages" ON)
 option(COMPRESS_MANPAGES "compress manpages" ON)
 option(GENERATE_DOC "generate API documentation" ON)
+if (GENERATE_DOC AND ENV{DO_COVERAGE})
+    message(STATUS "Not generating API documentation with DO_COVERAGE")
+endif()
 
 # {{{ Endianness
 include(TestBigEndian)
@@ -37,18 +40,26 @@ a_find_program(GIT_EXECUTABLE git FALSE)
 a_find_program(ASCIIDOC_EXECUTABLE asciidoc FALSE)
 a_find_program(XMLTO_EXECUTABLE xmlto FALSE)
 a_find_program(GZIP_EXECUTABLE gzip FALSE)
-# lua documentation
-a_find_program(LDOC_EXECUTABLE ldoc FALSE)
-if(NOT LDOC_EXECUTABLE)
-    a_find_program(LDOC_EXECUTABLE ldoc.lua FALSE)
-endif()
-if(LDOC_EXECUTABLE)
-    execute_process(COMMAND sh -c "${LDOC_EXECUTABLE} --sadly-ldoc-has-no-version-option 2>&1  | grep ' vs 1.4.5'"
-                    OUTPUT_VARIABLE LDOC_VERSION_RESULT)
-    if(NOT LDOC_VERSION_RESULT STREQUAL "")
-        message(WARNING "Ignoring LDoc, because version 1.4.5 is known to be broken")
-        unset(LDOC_EXECUTABLE CACHE)
+# Lua documentation
+if(GENERATE_DOC)
+    a_find_program(LDOC_EXECUTABLE ldoc FALSE)
+    if(NOT LDOC_EXECUTABLE)
+        a_find_program(LDOC_EXECUTABLE ldoc.lua FALSE)
     endif()
+    if(LDOC_EXECUTABLE)
+        execute_process(COMMAND sh -c "${LDOC_EXECUTABLE} --sadly-ldoc-has-no-version-option 2>&1  | grep ' vs 1.4.5'"
+                        OUTPUT_VARIABLE LDOC_VERSION_RESULT)
+        if(NOT LDOC_VERSION_RESULT STREQUAL "")
+            message(WARNING "Ignoring LDoc, because version 1.4.5 is known to be broken")
+            unset(LDOC_EXECUTABLE CACHE)
+        endif()
+    endif()
+    if(NOT LDOC_EXECUTABLE)
+        message(STATUS "Not generating API documentation. Missing: ldoc.")
+        set(GENERATE_DOC OFF)
+    endif()
+else()
+    message(STATUS "Not generating API documentation.")
 endif()
 # theme graphics
 a_find_program(CONVERT_EXECUTABLE convert TRUE)
@@ -75,13 +86,6 @@ if(GENERATE_MANPAGES)
         set(GENERATE_MANPAGES OFF)
     endif()
 endif()
-
-if(GENERATE_DOC)
-    if(NOT LDOC_EXECUTABLE)
-        message(STATUS "Not generating API documentation. Missing: ldoc")
-        set(GENERATE_DOC OFF)
-    endif()
-endif()
 # }}}
 
 # {{{ Version stamp
@@ -318,7 +322,6 @@ set(AWESOME_ICON_PATH        ${AWESOME_DATA_PATH}/icons)
 set(AWESOME_THEMES_PATH      ${AWESOME_DATA_PATH}/themes)
 # }}}
 
-
 if(GENERATE_DOC)
     # Load the common documentation
     include(docs/load_ldoc.cmake)
@@ -333,19 +336,25 @@ if(GENERATE_DOC)
 endif()
 
 # {{{ Configure files
-file(GLOB awesome_c_configure_files RELATIVE ${SOURCE_DIR}
+file(GLOB awesome_base_c_configure_files RELATIVE ${SOURCE_DIR}
     ${SOURCE_DIR}/*.c
-    ${SOURCE_DIR}/*.h
+    ${SOURCE_DIR}/*.h)
+
+file(GLOB awesome_c_configure_files RELATIVE ${SOURCE_DIR}
     ${SOURCE_DIR}/common/*.c
     ${SOURCE_DIR}/common/*.h
     ${SOURCE_DIR}/objects/*.c
     ${SOURCE_DIR}/objects/*.h)
+
 file(GLOB_RECURSE awesome_lua_configure_files RELATIVE ${SOURCE_DIR}
-    ${SOURCE_DIR}/lib/*.lua
+    ${SOURCE_DIR}/lib/*.lua)
+
+file(GLOB_RECURSE awesome_theme_configure_files RELATIVE ${SOURCE_DIR}
     ${SOURCE_DIR}/themes/*/*.lua)
+
 set(AWESOME_CONFIGURE_FILES
-    ${awesome_c_configure_files}
-    ${awesome_lua_configure_files}
+    ${awesome_base_c_configure_files}
+    ${awesome_theme_configure_files}
     config.h
     docs/config.ld
     awesome-version-internal.h)
@@ -356,6 +365,33 @@ foreach(file ${AWESOME_CONFIGURE_FILES})
                    ESCAPE_QUOTES
                    @ONLY)
 endforeach()
+
+set(AWESOME_CONFIGURE_COPYONLY_WITHCOV_FILES
+    ${awesome_c_configure_files}
+    ${awesome_lua_configure_files}
+)
+
+if(DO_COVERAGE)
+    foreach(file ${AWESOME_CONFIGURE_COPYONLY_WITHCOV_FILES})
+        configure_file(${SOURCE_DIR}/${file}
+                    ${BUILD_DIR}/${file}
+                    COPYONLY)
+    endforeach()
+
+    # There is no way to avoid that one
+    configure_file(${SOURCE_DIR}/lib/awful/util.lua
+                    ${BUILD_DIR}/lib/awful/util.lua
+                    ESCAPE_QUOTES
+                    @ONLY)
+else()
+    foreach(file ${AWESOME_CONFIGURE_COPYONLY_WITHCOV_FILES})
+        configure_file(${SOURCE_DIR}/${file}
+                    ${BUILD_DIR}/${file}
+                    ESCAPE_QUOTES
+                    @ONLY)
+    endforeach()
+endif()
+
 #}}}
 
 # {{{ Generate some aggregated documentation from lua script
@@ -391,14 +427,4 @@ foreach(file ${AWESOME_ADDITIONAL_FILES})
 endforeach()
 #}}}
 
-# The examples coverage need to be done again after the configure_file has
-# inserted the additional code. Otherwise, the result will be off, rendering
-# the coverage useless as a tool to track untested code.
-if(GENERATE_DOC AND DO_COVERAGE)
-    message(STATUS "Running tests again with coverage")
-    set(USE_LCOV 1)
-
-    include(tests/examples/CMakeLists.txt)
-endif()
-
 # vim: filetype=cmake:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80:foldmethod=marker

spec/menubar/icon_theme_spec.lua

@@ -8,8 +8,8 @@ local string = string
 local icon_theme = require("menubar.icon_theme")
 
 local base_directories = {
-    os.getenv("PWD") .. "/spec/menubar/icons",
-    os.getenv("PWD") .. "/icons"
+    (os.getenv("SOURCE_DIRECTORY") or '.') .. "/spec/menubar/icons",
+    (os.getenv("SOURCE_DIRECTORY") or '.') .. "/icons"
 }
 
 describe("menubar.icon_theme find_icon_path", function()

tests/examples/CMakeLists.txt

@@ -14,20 +14,42 @@ cmake_minimum_required(VERSION 3.0.0)
 # Get and update the LUA_PATH so the scripts can be executed without Awesome.
 execute_process(COMMAND lua -e print\(package.path\) OUTPUT_VARIABLE "LUA_PATH_")
 
+# Make sure the system can be called from the test directory
+if(NOT SOURCE_DIR AND ${CMAKE_CURRENT_SOURCE_DIR} MATCHES "/tests/examples")
+    get_filename_component(TOP_SOURCE_DIR
+                           "${CMAKE_CURRENT_SOURCE_DIR}/../.." ABSOLUTE)
+    # Used by .luacov.
+    set(ENV{SOURCE_DIRECTORY} ${TOP_SOURCE_DIR})
+else()
+    set(TOP_SOURCE_DIR ${CMAKE_SOURCE_DIR})
+endif()
+
+if ($ENV{DO_COVERAGE})
+  execute_process(
+    COMMAND lua -e "require('luacov.runner')('${TOP_SOURCE_DIR}/.luacov')"
+    RESULT_VARIABLE TEST_RESULT
+    ERROR_VARIABLE  TEST_ERROR
+    ERROR_STRIP_TRAILING_WHITESPACE)
+  if (TEST_RESULT OR NOT TEST_ERROR STREQUAL "")
+    message(${TEST_ERROR})
+    message(FATAL_ERROR "Failed to run luacov.runner.")
+  endif()
+endif()
+
 # Add the main awesome lua libraries.
 set(LUA_PATH2_ "\
-${CMAKE_SOURCE_DIR}/lib/?.lua;\
-${CMAKE_SOURCE_DIR}/lib/?/init.lua;\
-${CMAKE_SOURCE_DIR}/lib/?;\
-${CMAKE_SOURCE_DIR}/themes/?.lua;\
-${CMAKE_SOURCE_DIR}/themes/?;"
+${TOP_SOURCE_DIR}/lib/?.lua;\
+${TOP_SOURCE_DIR}/lib/?/init.lua;\
+${TOP_SOURCE_DIR}/lib/?;\
+${TOP_SOURCE_DIR}/themes/?.lua;\
+${TOP_SOURCE_DIR}/themes/?;"
 )
 
 # Add the C API shims.
 set(LUA_PATH3_ "\
-${CMAKE_SOURCE_DIR}/tests/examples/shims/?.lua;\
-${CMAKE_SOURCE_DIR}/tests/examples/shims/?/init.lua;\
-${CMAKE_SOURCE_DIR}/tests/examples/shims/?;"
+${TOP_SOURCE_DIR}/tests/examples/shims/?.lua;\
+${TOP_SOURCE_DIR}/tests/examples/shims/?/init.lua;\
+${TOP_SOURCE_DIR}/tests/examples/shims/?;"
 )
 
 # Done in 3 variables to avoid CMake from implicitly converting into a list.
@@ -133,7 +155,7 @@ endfunction()
 # Get the namespace of a file.
 function(get_namespace result_variable file)
     get_filename_component(path "${file}" DIRECTORY)
-    string(LENGTH "${SOURCE_DIR}/tests/examples" prefix_length)
+    string(LENGTH "${TOP_SOURCE_DIR}/tests/examples" prefix_length)
     string(REPLACE "/" "_" namespace "${path}")
     string(SUBSTRING "${namespace}" "${prefix_length}" -1 namespace)
 
@@ -148,16 +170,11 @@ function(run_test test_path namespace escaped_content)
     get_filename_component(${test_path} TEST_FILE_NAME NAME)
     set(IMAGE_PATH "${IMAGE_DIR}/AUTOGEN${namespace}_${TEST_FILE_NAME}")
 
-    # Use the example testing as coverage source.
-    if (USE_LCOV)
-        set(LCOV_PATH ${SOURCE_DIR}/.luacov)
-    endif()
-
     # Execute the script, leave the image extension decision to the test.
     # SVG is preferred, but PNG is better suited for some tests, like bitmap
     # patterns.
     execute_process(
-        COMMAND lua ${template} ${test_path} ${IMAGE_PATH} ${LCOV_PATH}
+        COMMAND lua ${template} ${test_path} ${IMAGE_PATH}
         OUTPUT_VARIABLE TEST_OUTPUT
         ERROR_VARIABLE  TEST_ERROR
     )
@@ -231,12 +248,12 @@ endfunction()
 
 # Find and run all test files.
 file(GLOB_RECURSE test_files LIST_DIRECTORIES false
-    "${SOURCE_DIR}/tests/examples/*.lua")
+    "${TOP_SOURCE_DIR}/tests/examples/*.lua")
 foreach(file ${test_files})
     if ((NOT "${file}" MATCHES ".*/shims/.*")
         AND (NOT "${file}" MATCHES ".*/template.lua"))
 
-        file(RELATIVE_PATH relative_file "${SOURCE_DIR}" "${file}")
+        file(RELATIVE_PATH relative_file "${TOP_SOURCE_DIR}" "${file}")
         message(STATUS "Running ${relative_file}...")
 
         # Get the file name without the extension.

tests/examples/shims/_common_template.lua

@@ -1,4 +1,4 @@
-return function(_, _, luacovpath)
+return function(_, _)
 
     -- Set the global shims
     -- luacheck: globals awesome root tag screen client mouse drawin button
@@ -15,9 +15,9 @@ return function(_, _, luacovpath)
     assert(awesome and root and tag and screen and client and mouse)
 
     -- If luacov is available, use it. Else, do nothing.
-    pcall(function()
-        require("luacov.runner")(luacovpath)
-    end)
+    if (os.getenv('DO_COVERAGE') or '0') ~= '0' then
+        require('luacov.runner')(os.getenv('SOURCE_DIRECTORY')..'/.luacov')
+    end
 
     -- Silence debug warnings
     require("gears.debug").print_warning = function() end

tests/run.sh

@@ -22,8 +22,8 @@ fi
 
 # Change to file's dir (POSIXly).
 cd -P -- "$(dirname -- "$0")"
-this_dir=$PWD
-source_dir="$PWD/.."
+this_dir="$PWD"
+source_dir="${this_dir%/*}"
 
 # Either the build dir is passed in $CMAKE_BINARY_DIR or we guess based on $PWD
 build_dir="$CMAKE_BINARY_DIR"
@@ -179,7 +179,8 @@ fi
 start_awesome() {
     cd "$build_dir"
     # Kill awesome after $timeout_stale seconds (e.g. for errors during test setup).
-    DISPLAY="$D" timeout "$timeout_stale" "$AWESOME" -c "$RC_FILE" "${awesome_options[@]}" > "$awesome_log" 2>&1 &
+    # SOURCE_DIRECTORY is used by .luacov.
+    DISPLAY="$D" SOURCE_DIRECTORY="$source_dir" timeout "$timeout_stale" "$AWESOME" -c "$RC_FILE" "${awesome_options[@]}" > "$awesome_log" 2>&1 &
     awesome_pid=$!
     cd - >/dev/null