diff --git a/tests/examples/CMakeLists.txt b/tests/examples/CMakeLists.txt
index 4cb869f6e..2f45b84bc 100644
--- a/tests/examples/CMakeLists.txt
+++ b/tests/examples/CMakeLists.txt
@@ -188,6 +188,14 @@ function(run_test test_path namespace escaped_content)
         set(DOC_BLOCK_PREFIX "@usage")
     endif()
 
+    # Does the text generate text output?
+    if(tmp_content MATCHES "--DOC_GEN_OUTPUT")
+        # The expected output is next to the .lua file in a .output.txt file
+        string(REPLACE ".lua" ".output.txt" expected_output_path ${test_path})
+    else()
+        set(expected_output_path "/dev/null")
+    endif()
+
     # Get the file name without the extension.
     get_filename_component(${test_path} TEST_FILE_NAME NAME)
     set(IMAGE_PATH "${IMAGE_DIR}/AUTOGEN${namespace}_${TEST_FILE_NAME}")
@@ -197,12 +205,12 @@ function(run_test test_path namespace escaped_content)
     message(STATUS "Running ${rel_test_path}…")
 
     execute_process(
-        COMMAND ${LUA_COV_RUNNER} ${template} ${test_path} ${IMAGE_PATH}
+        COMMAND "${TOP_SOURCE_DIR}/tests/examples/runner.sh" "${expected_output_path}" ${LUA_COV_RUNNER} ${template} ${test_path} ${IMAGE_PATH}
         RESULT_VARIABLE TEST_RESULT
         OUTPUT_VARIABLE TEST_OUTPUT
         ERROR_VARIABLE  TEST_ERROR
     )
-    if (TEST_RESULT OR NOT TEST_ERROR STREQUAL "")
+    if (TEST_RESULT OR NOT TEST_OUTPUT STREQUAL "" OR NOT TEST_ERROR STREQUAL "")
         message("Result: ${TEST_RESULT}")
         if (NOT TEST_OUTPUT STREQUAL "")
             message("Output: ${TEST_OUTPUT}")
@@ -236,12 +244,7 @@ function(run_test test_path namespace escaped_content)
     # If there is an output, assume it is relevant and add it to the
     # documentation under the image.
     if(tmp_content MATCHES "--DOC_GEN_OUTPUT")
-        # The expected output is next to the .lua file in a .output.txt file
-        string(REPLACE ".lua" ".output.txt" expected_output_path ${test_path})
         file(READ "${expected_output_path}" expected_output)
-        if(NOT "${TEST_OUTPUT}" STREQUAL "${expected_output}")
-            message(SEND_ERROR "Wrong output from ${test_path}:\nExpected:\n${expected_output}\nGot:\n${TEST_OUTPUT}")
-        endif()
 
         set(TEST_DOC_CONTENT
             "${TEST_DOC_CONTENT}\n${DOC_LINE_PREFIX}\n${DOC_LINE_PREFIX}**Usage example output**:\n${DOC_LINE_PREFIX}"
@@ -249,14 +252,10 @@ function(run_test test_path namespace escaped_content)
 
         # Markdown requires an empty line before and after, and 4 spaces.
         escape_string(
-            "\n${TEST_OUTPUT}"
+            "\n${expected_output}"
             "${TEST_DOC_CONTENT}" TEST_DOC_CONTENT "    "
         )
         set(TEST_DOC_CONTENT "${TEST_DOC_CONTENT}\n${DOC_LINE_PREFIX}")
-    else()
-        if(NOT ${TEST_OUTPUT} STREQUAL "")
-            message(FATAL_ERROR "Unexpected output from ${test_path}: ${TEST_OUTPUT}")
-        endif()
     endif()
 
     # If there is some @* content, append it.
diff --git a/tests/examples/runner.sh b/tests/examples/runner.sh
new file mode 100755
index 000000000..d3eb820c9
--- /dev/null
+++ b/tests/examples/runner.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+# Todo: Give this shell script a sane calling convention. Right now it has a
+# file containing the expected output as first argument and just executes
+# everything else.
+
+set -ue
+
+# Cleanup on errors / aborting
+cleanup() {
+    rm -rf "$file_stderr" "$file_stdout" || true
+}
+trap "cleanup" 0 2 3 15
+
+file_stdout=$(mktemp)
+file_stderr=$(mktemp)
+
+expected_output=$1
+shift
+
+# Run the command that we were given
+exit_code=0
+"$@" > ${file_stdout} 2> ${file_stderr} || exit_code=$?
+
+# If exit code is not zero or anything was produced on stderr...
+if [ $exit_code -ne 0 -o -s "${file_stderr}" ]
+then
+    echo "Result: ${exit_code}"
+    if [ -s "${file_stdout}" ]
+    then
+        echo "Output:"
+        cat "${file_stdout}"
+    fi
+    if [ -s "${file_stderr}" ]
+    then
+        echo "Error:"
+        cat "${file_stderr}"
+    fi
+    exit 1
+fi
+
+# Check if we got the output we wanted
+if ! cmp --silent "${file_stdout}" "${expected_output}"
+then
+    echo "Expected text from ${expected_output}, but got:"
+    diff -u "${expected_output}" "${file_stdout}" || true
+    exit 1
+fi
+
+exit 0
+
+# vim: filetype=sh:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80