diff --git a/tests/_runner.lua b/tests/_runner.lua
index 86dedecb4..56860de27 100644
--- a/tests/_runner.lua
+++ b/tests/_runner.lua
@@ -16,6 +16,20 @@ runner.add_to_default_rules = function(r)
   table.insert(awful.rules.rules, r)
 end
 
+-- Was the runner started already?
+local running = false
+
+-- This is used if a test causes errors before starting the runner
+timer.start_new(1, function()
+    if not running then
+        io.stderr:write("Error: run_steps() was never called\n")
+        if not runner.quit_awesome_on_error then
+            io.stderr:write("Keeping awesome open...\n")
+            return  -- keep awesome open on error.
+        end
+        awesome.quit()
+    end
+end)
 
 runner.run_steps = function(steps)
   -- Setup timer/timeout to limit waiting for signal and quitting awesome.
@@ -24,6 +38,8 @@ runner.run_steps = function(steps)
   local wait=20
   local step=1
   local step_count=0
+  assert(not running, "run_steps() was called twice")
+  running = true
   t:connect_signal("timeout", function() timer.delayed_call(function()
     io.flush()  -- for "tail -f".
     step_count = step_count + 1
diff --git a/tests/test-awful-widget-watch.lua b/tests/test-awful-widget-watch.lua
index f0c03d327..0a88583aa 100644
--- a/tests/test-awful-widget-watch.lua
+++ b/tests/test-awful-widget-watch.lua
@@ -1,5 +1,6 @@
 --- Test for awful.widget.watch
 
+local runner = require("_runner")
 local watch = require("awful.widget.watch")
 
 local callbacks_done = 0
@@ -25,6 +26,6 @@ local steps = {
         end
     end
 }
-require("_runner").run_steps(steps)
+runner.run_steps(steps)
 
 -- vim: filetype=lua:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80
diff --git a/tests/test-benchmark.lua b/tests/test-benchmark.lua
index ca35f171f..c725de976 100644
--- a/tests/test-benchmark.lua
+++ b/tests/test-benchmark.lua
@@ -1,6 +1,7 @@
 -- Some benchmarks that aren't really tests, but are included here anyway so
 -- that we notice if they break.
 
+local runner = require("_runner")
 local awful = require("awful")
 local GLib = require("lgi").GLib
 local create_wibox = require("_wibox_helper").create_wibox
@@ -75,6 +76,6 @@ benchmark(relayout_textclock, "relayout textclock")
 benchmark(redraw_textclock, "redraw textclock")
 benchmark(e2e_tag_switch, "tag switch")
 
-require("_runner").run_steps({ function() return true end })
+runner.run_steps({ function() return true end })
 
 -- vim: filetype=lua:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80
diff --git a/tests/test-focus.lua b/tests/test-focus.lua
index 084be2fbd..41b3cd561 100644
--- a/tests/test-focus.lua
+++ b/tests/test-focus.lua
@@ -1,6 +1,7 @@
 --- Tests for focus signals / property.
 -- Test for https://github.com/awesomeWM/awesome/issues/134.
 
+local runner = require("_runner")
 local awful = require("awful")
 
 local beautiful = require("beautiful")
@@ -44,6 +45,6 @@ local steps = {
   end
 }
 
-require("_runner").run_steps(steps)
+runner.run_steps(steps)
 
 -- vim: filetype=lua:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80
diff --git a/tests/test-leak-client.lua b/tests/test-leak-client.lua
index c70bf3699..c6d209bf4 100644
--- a/tests/test-leak-client.lua
+++ b/tests/test-leak-client.lua
@@ -1,4 +1,5 @@
 -- Some memory leak checks involving clients as integration tests.
+local runner = require("_runner")
 local awful = require("awful")
 local wibox = require("wibox")
 
@@ -78,6 +79,6 @@ local steps = {
     end,
 }
 
-require("_runner").run_steps(steps)
+runner.run_steps(steps)
 
 -- vim: filetype=lua:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80
diff --git a/tests/test-leaks.lua b/tests/test-leaks.lua
index e271bc55a..d12d652e9 100644
--- a/tests/test-leaks.lua
+++ b/tests/test-leaks.lua
@@ -1,5 +1,6 @@
 -- Some memory leak checks as integration tests.
 
+local runner = require("_runner")
 local awful = require("awful")
 local cairo = require("lgi").cairo
 local create_wibox = require("_wibox_helper").create_wibox
@@ -77,6 +78,6 @@ collectable(awful.widget.tasklist(1, awful.widget.tasklist.filter.currenttags))
 prepare_for_collect = emit_refresh
 collectable(create_wibox())
 
-require("_runner").run_steps({ function() return true end })
+runner.run_steps({ function() return true end })
 
 -- vim: filetype=lua:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80
diff --git a/tests/test-spawn-snid.lua b/tests/test-spawn-snid.lua
index 27738fc29..2aaadc084 100644
--- a/tests/test-spawn-snid.lua
+++ b/tests/test-spawn-snid.lua
@@ -1,5 +1,6 @@
 --- Tests for spawn's startup notifications.
 
+local runner = require("_runner")
 local spawn = require("awful.spawn")
 
 local manage_called, c_snid
@@ -38,6 +39,6 @@ local steps = {
   end
 }
 
-require("_runner").run_steps(steps)
+runner.run_steps(steps)
 
 -- vim: filetype=lua:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80
diff --git a/tests/test-spawn.lua b/tests/test-spawn.lua
index d2f0a1aa7..63d87c155 100644
--- a/tests/test-spawn.lua
+++ b/tests/test-spawn.lua
@@ -1,5 +1,6 @@
 --- Tests for spawn
 
+local runner = require("_runner")
 local spawn = require("awful.spawn")
 
 local spawns_done = 0
@@ -63,6 +64,6 @@ local steps = {
   end,
 }
 
-require("_runner").run_steps(steps)
+runner.run_steps(steps)
 
 -- vim: filetype=lua:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80