Travis: check each commit separately (for the sake of git-bisect) (#936)

Previous commits are only tested within one build (luajit, which appears
to be the fastest, maybe/also because it does no coverage).

Fixes https://github.com/awesomeWM/awesome/issues/935.

.travis.yml

@@ -4,10 +4,10 @@ language: c
 env:
   matrix:
     - LUA=5.2 LUANAME=lua5.2 BUILD_APIDOC=true DO_COVERAGE=coveralls
+    # luajit: installed from source.
+    - LUA=5.1 LUANAME=luajit-2.0 LUALIBRARY=/usr/lib/libluajit-5.1.so LUAROCKS_ARGS=--lua-suffix=jit-2.0.4 TEST_PREV_COMMITS=1
     # Note: luarocks does not work with Lua 5.0.
     - LUA=5.1 LUANAME=lua5.1 BUILD_IN_DIR=/tmp/awesome-build
-    # luajit: installed from source.
-    - LUA=5.1 LUANAME=luajit-2.0 LUALIBRARY=/usr/lib/libluajit-5.1.so LUAROCKS_ARGS=--lua-suffix=jit-2.0.4
     # Lua 5.2 with older lgi and screen size not divisible by 2.
     - LUA=5.2 LUANAME=lua5.2 LGIVER=0.7.1 TESTS_SCREEN_SIZE=1921x1079
     # Lua 5.3 isn't available in Ubuntu Trusty, so some magic below installs it.
@@ -102,16 +102,42 @@ install:
       fi
     }
 
+  # Functions for custom Travis folds.
+  - |
+    travis_fold_start() {
+      echo -n "travis_"
+      echo "fold:start:$1"
+      travis_fold_current="$1"
+    }
+    travis_fold_end() {
+      echo -n "travis_"
+      echo "fold:end:$travis_fold_current"
+    }
+    travis_run_in_fold() {
+      travis_fold_start "$1"
+      shift
+      "$@"
+      local result="$?"
+      travis_fold_end
+      return $result
+    }
 script:
   - export CMAKE_ARGS="-DLUA_LIBRARY=${LUALIBRARY} -DLUA_INCLUDE_DIR=${LUAINCLUDE} -D OVERRIDE_VERSION=$AWESOME_VERSION -D DO_COVERAGE=${DO_COVERAGE}"
   - |
     if [ -n "$BUILD_IN_DIR" ]; then
+      # Explicitly remove the Makefile to not build from the src dir accidentally.
+      rm Makefile
       SOURCE_DIRECTORY="$PWD"
       mkdir "$BUILD_IN_DIR"
       cd "$BUILD_IN_DIR"
-        cmake $CMAKE_ARGS "$SOURCE_DIRECTORY"
+      travis_run_in_fold "build_in_dir" cmake $CMAKE_ARGS "$SOURCE_DIRECTORY"
+    fi
+  - travis_run_in_fold "make" make
+  - |
+    if [ "$TRAVIS_TEST_RESULT" = 0 ]; then
+      travis_run_in_fold "make.install" sudo env PATH=$PATH make install
+      awesome --version
     fi
-  - make && sudo env PATH=$PATH make install && awesome --version
   - |
     if [ "$TRAVIS_TEST_RESULT" = 0 ]; then
       do_codecov samples
@@ -119,13 +145,44 @@ script:
   - |
     if [ "$TRAVIS_TEST_RESULT" = 0 ]; then
       if [ "$DO_COVERAGE" != "0" ]; then
-        make check-coverage || exit 1
+        travis_fold_start "DO_COVERAGE"
-        do_codecov unittests
+        (make check-coverage \
-        sed -i "1 i\\require('luacov.runner')('"$PWD"/.luacov')" build/awesomerc.lua || exit 1
+         && do_codecov unittests \
-        BUILD_DIRECTORY="" tests/run.sh || exit 1
+         && sed -i "1 i\\require('luacov.runner')('"$PWD"/.luacov')" build/awesomerc.lua \
-        do_codecov functionaltests
+         && BUILD_DIRECTORY="" tests/run.sh \
+         && do_codecov functionaltests)
+        ret=$?
+        travis_fold_end
+        exit $ret
       else
-        make check
+        travis_run_in_fold "make.check" make check
+      fi
+    fi
+  - |
+    if [ "$TEST_PREV_COMMITS" = 1 ] && ! [ "$TRAVIS_PULL_REQUEST" = false ]; then
+      # Check each commit separately (to make git-bisect less annoying).
+      echo "Testing previous commits based on TRAVIS_COMMIT_RANGE=$TRAVIS_COMMIT_RANGE"
+      rev_list="$(git rev-list --bisect-all $TRAVIS_COMMIT_RANGE)"
+      echo "rev-list: $rev_list"
+      commits="$(echo $rev_list | grep -v 'dist=0' | cut -d\  -f 1)"
+      echo "Testing commits: $commits"
+      n="$(echo commits | wc -l)"
+      i=0
+      failed=
+      for commit in $commits; do
+        i=$((i+1))
+        travis_fold_start "test_commit_${commit}_.$i.$n"
+        echo "Testing commit $commit"
+        git checkout "$commit"
+        git --no-pager show --stat
+        if ! make all check; then
+          failed="$failed $commit"
+        fi
+        travis_fold_end
+      done
+      if [ -n "$failed" ]; then
+        echo "Checks failed for these commits:$failed"
+        exit 1
       fi
     fi