diff --git a/CMakeLists.txt b/CMakeLists.txt
index 84218c45..8b86c514 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -401,12 +401,11 @@ add_custom_target(check-integration
     USES_TERMINAL)
 add_dependencies(check-integration test-gravity)
 add_custom_target(check-themes
-    ${CMAKE_SOURCE_DIR}/tests/themes/run.sh
-    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+    ${CMAKE_COMMAND} -E env CMAKE_BINARY_DIR='${CMAKE_BINARY_DIR}' ./tests/themes/run.sh
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
     COMMENT "Testing themes"
     USES_TERMINAL
-    DEPENDS generated_icons generate_awesomerc ${PROJECT_AWE_NAME}
-    VERBATIM)
+    DEPENDS generated_icons generate_awesomerc ${PROJECT_AWE_NAME})
 list(APPEND CHECK_TARGETS check-themes)
 
 add_custom_target(check-requires
diff --git a/tests/themes/run.sh b/tests/themes/run.sh
index 41a2a6b4..21c7d1db 100755
--- a/tests/themes/run.sh
+++ b/tests/themes/run.sh
@@ -6,14 +6,19 @@
 
 set -e
 
-if ! [ -f CMakeCache.txt ]; then
-  echo 'This should be run from the build directory (expected CMakeCache.txt).' >&2
+if ! [ -f CMakeLists.txt ]; then
+  echo 'This should be run from the source directory (expected CMakeLists.txt).' >&2
   exit 64
 fi
-build_dir="$PWD"
-tests_dir="$(dirname -- "$0")/.."
+source_dir="$PWD"
 
-config_file="$build_dir/test-themes-awesomerc.lua"
+config_file="$(mktemp)"
+
+# Cleanup on errors / aborting.
+cleanup() {
+    rm "$config_file" || true
+}
+trap "cleanup" 0 2 3 15
 
 for theme_file in themes/*/theme.lua; do
   echo "==== Testing theme: $theme_file ===="
@@ -22,10 +27,8 @@ for theme_file in themes/*/theme.lua; do
 
   sed "s~default/theme~$theme_name/theme~g" "awesomerc.lua" > "$config_file"
 
-  # Set CMAKE_BINARY_DIR for out-of-tree builds.
-  CMAKE_BINARY_DIR="$PWD" \
   AWESOME_RC_FILE="$config_file" \
-    AWESOME_THEMES_PATH="$PWD/themes" \
+    AWESOME_THEMES_PATH="$source_dir/themes" \
     AWESOME_ICON_PATH="$PWD/icons" \
-    "$tests_dir/run.sh" themes/tests.lua
+    "$source_dir/tests/run.sh" themes/tests.lua
 done