test runner: Move running to a shell script

Instead of running the example tests directly from CMake and checking
the results via CMake, the tests are now run through a shell script.

So far, there are too many variables involved for me to easily figure
out how to run this shell script in the building phase instead of the
configuring phase, but at least this commit moves the "actual running"
out of CMake, bringing us a step closer to that goal.

Signed-off-by: Uli Schlachter <psychon@znc.in>

tests/examples/CMakeLists.txt

@@ -188,6 +188,14 @@ function(run_test test_path namespace escaped_content)
         set(DOC_BLOCK_PREFIX "@usage")
     endif()
 
+    # Does the text generate text output?
+    if(tmp_content MATCHES "--DOC_GEN_OUTPUT")
+        # The expected output is next to the .lua file in a .output.txt file
+        string(REPLACE ".lua" ".output.txt" expected_output_path ${test_path})
+    else()
+        set(expected_output_path "/dev/null")
+    endif()
+
     # Get the file name without the extension.
     get_filename_component(${test_path} TEST_FILE_NAME NAME)
     set(IMAGE_PATH "${IMAGE_DIR}/AUTOGEN${namespace}_${TEST_FILE_NAME}")
@@ -197,12 +205,12 @@ function(run_test test_path namespace escaped_content)
     message(STATUS "Running ${rel_test_path}…")
 
     execute_process(
-        COMMAND ${LUA_COV_RUNNER} ${template} ${test_path} ${IMAGE_PATH}
+        COMMAND "${TOP_SOURCE_DIR}/tests/examples/runner.sh" "${expected_output_path}" ${LUA_COV_RUNNER} ${template} ${test_path} ${IMAGE_PATH}
         RESULT_VARIABLE TEST_RESULT
         OUTPUT_VARIABLE TEST_OUTPUT
         ERROR_VARIABLE  TEST_ERROR
     )
-    if (TEST_RESULT OR NOT TEST_ERROR STREQUAL "")
+    if (TEST_RESULT OR NOT TEST_OUTPUT STREQUAL "" OR NOT TEST_ERROR STREQUAL "")
         message("Result: ${TEST_RESULT}")
         if (NOT TEST_OUTPUT STREQUAL "")
             message("Output: ${TEST_OUTPUT}")
@@ -236,12 +244,7 @@ function(run_test test_path namespace escaped_content)
     # If there is an output, assume it is relevant and add it to the
     # documentation under the image.
     if(tmp_content MATCHES "--DOC_GEN_OUTPUT")
-        # The expected output is next to the .lua file in a .output.txt file
-        string(REPLACE ".lua" ".output.txt" expected_output_path ${test_path})
         file(READ "${expected_output_path}" expected_output)
-        if(NOT "${TEST_OUTPUT}" STREQUAL "${expected_output}")
-            message(SEND_ERROR "Wrong output from ${test_path}:\nExpected:\n${expected_output}\nGot:\n${TEST_OUTPUT}")
-        endif()
 
         set(TEST_DOC_CONTENT
             "${TEST_DOC_CONTENT}\n${DOC_LINE_PREFIX}\n${DOC_LINE_PREFIX}**Usage example output**:\n${DOC_LINE_PREFIX}"
@@ -249,14 +252,10 @@ function(run_test test_path namespace escaped_content)
 
         # Markdown requires an empty line before and after, and 4 spaces.
         escape_string(
-            "\n${TEST_OUTPUT}"
+            "\n${expected_output}"
             "${TEST_DOC_CONTENT}" TEST_DOC_CONTENT "    "
         )
         set(TEST_DOC_CONTENT "${TEST_DOC_CONTENT}\n${DOC_LINE_PREFIX}")
-    else()
-        if(NOT ${TEST_OUTPUT} STREQUAL "")
-            message(FATAL_ERROR "Unexpected output from ${test_path}: ${TEST_OUTPUT}")
-        endif()
     endif()
 
     # If there is some @* content, append it.

tests/examples/runner.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+# Todo: Give this shell script a sane calling convention. Right now it has a
+# file containing the expected output as first argument and just executes
+# everything else.
+
+set -ue
+
+# Cleanup on errors / aborting
+cleanup() {
+    rm -rf "$file_stderr" "$file_stdout" || true
+}
+trap "cleanup" 0 2 3 15
+
+file_stdout=$(mktemp)
+file_stderr=$(mktemp)
+
+expected_output=$1
+shift
+
+# Run the command that we were given
+exit_code=0
+"$@" > ${file_stdout} 2> ${file_stderr} || exit_code=$?
+
+# If exit code is not zero or anything was produced on stderr...
+if [ $exit_code -ne 0 -o -s "${file_stderr}" ]
+then
+    echo "Result: ${exit_code}"
+    if [ -s "${file_stdout}" ]
+    then
+        echo "Output:"
+        cat "${file_stdout}"
+    fi
+    if [ -s "${file_stderr}" ]
+    then
+        echo "Error:"
+        cat "${file_stderr}"
+    fi
+    exit 1
+fi
+
+# Check if we got the output we wanted
+if ! cmp --silent "${file_stdout}" "${expected_output}"
+then
+    echo "Expected text from ${expected_output}, but got:"
+    diff -u "${expected_output}" "${file_stdout}" || true
+    exit 1
+fi
+
+exit 0
+
+# vim: filetype=sh:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80