From b5ca8bf93779bd5fef55980f7608c0c9131ca362 Mon Sep 17 00:00:00 2001 From: Uli Schlachter Date: Wed, 25 Jul 2018 16:03:26 +0200 Subject: [PATCH 1/2] example tests: Save expected test output in a txt file This commit adds a .txt file next to each example test that generates a text output. This text file contains the expected output and it is an error if the actual output does not match the expected output. This means that we no longer have to run the example tests before we can expand all the @foo@ expressions that occur. While touching this, I also fixed some typos and unexpected newlines in the tests' output. Signed-off-by: Uli Schlachter --- tests/examples/CMakeLists.txt | 15 ++++++++++++--- tests/examples/awful/mouse/coords.output.txt | 1 + .../awful/placement/closest_mouse.output.txt | 1 + .../awful/placement/no_offscreen.output.txt | 2 ++ tests/examples/text/gears/object/properties.lua | 2 +- .../text/gears/object/properties.output.txt | 9 +++++++++ .../examples/text/gears/object/signal.output.txt | 2 ++ .../text/gears/sort/topological.output.txt | 6 ++++++ tests/examples/wibox/layout/grid/add.output.txt | 1 + .../wibox/layout/grid/extend_column.output.txt | 1 + .../wibox/layout/grid/insert_column.output.txt | 1 + tests/examples/wibox/layout/grid/orientation.lua | 3 +-- .../wibox/layout/grid/orientation.output.txt | 8 ++++++++ .../examples/wibox/layout/grid/remove.output.txt | 1 + .../wibox/layout/grid/remove_column.output.txt | 1 + 15 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 tests/examples/awful/mouse/coords.output.txt create mode 100644 tests/examples/awful/placement/closest_mouse.output.txt create mode 100644 tests/examples/awful/placement/no_offscreen.output.txt create mode 100644 tests/examples/text/gears/object/properties.output.txt create mode 100644 tests/examples/text/gears/object/signal.output.txt create mode 100644 tests/examples/text/gears/sort/topological.output.txt create mode 100644 tests/examples/wibox/layout/grid/add.output.txt create mode 100644 tests/examples/wibox/layout/grid/extend_column.output.txt create mode 100644 tests/examples/wibox/layout/grid/insert_column.output.txt create mode 100644 tests/examples/wibox/layout/grid/orientation.output.txt create mode 100644 tests/examples/wibox/layout/grid/remove.output.txt create mode 100644 tests/examples/wibox/layout/grid/remove_column.output.txt diff --git a/tests/examples/CMakeLists.txt b/tests/examples/CMakeLists.txt index 685147569..4cb869f6e 100644 --- a/tests/examples/CMakeLists.txt +++ b/tests/examples/CMakeLists.txt @@ -232,12 +232,17 @@ function(run_test test_path namespace escaped_content) set(OUTPUT_IMAGE_PATH "") endif() + # Does the text generate text output? # If there is an output, assume it is relevant and add it to the # documentation under the image. - if(NOT ${TEST_OUTPUT} STREQUAL "") - if(NOT tmp_content MATCHES "--DOC_GEN_OUTPUT") - message(FATAL_ERROR "Unexpected output from ${test_path}: ${TEST_OUTPUT}") + if(tmp_content MATCHES "--DOC_GEN_OUTPUT") + # The expected output is next to the .lua file in a .output.txt file + string(REPLACE ".lua" ".output.txt" expected_output_path ${test_path}) + file(READ "${expected_output_path}" expected_output) + if(NOT "${TEST_OUTPUT}" STREQUAL "${expected_output}") + message(SEND_ERROR "Wrong output from ${test_path}:\nExpected:\n${expected_output}\nGot:\n${TEST_OUTPUT}") endif() + set(TEST_DOC_CONTENT "${TEST_DOC_CONTENT}\n${DOC_LINE_PREFIX}\n${DOC_LINE_PREFIX}**Usage example output**:\n${DOC_LINE_PREFIX}" ) @@ -248,6 +253,10 @@ function(run_test test_path namespace escaped_content) "${TEST_DOC_CONTENT}" TEST_DOC_CONTENT " " ) set(TEST_DOC_CONTENT "${TEST_DOC_CONTENT}\n${DOC_LINE_PREFIX}") + else() + if(NOT ${TEST_OUTPUT} STREQUAL "") + message(FATAL_ERROR "Unexpected output from ${test_path}: ${TEST_OUTPUT}") + endif() endif() # If there is some @* content, append it. diff --git a/tests/examples/awful/mouse/coords.output.txt b/tests/examples/awful/mouse/coords.output.txt new file mode 100644 index 000000000..f8c9d43a6 --- /dev/null +++ b/tests/examples/awful/mouse/coords.output.txt @@ -0,0 +1 @@ +235 diff --git a/tests/examples/awful/placement/closest_mouse.output.txt b/tests/examples/awful/placement/closest_mouse.output.txt new file mode 100644 index 000000000..ff6126c6e --- /dev/null +++ b/tests/examples/awful/placement/closest_mouse.output.txt @@ -0,0 +1 @@ +Closest corner: top_left diff --git a/tests/examples/awful/placement/no_offscreen.output.txt b/tests/examples/awful/placement/no_offscreen.output.txt new file mode 100644 index 000000000..e6857e61c --- /dev/null +++ b/tests/examples/awful/placement/no_offscreen.output.txt @@ -0,0 +1,2 @@ +Before: x=-30, y=-30, width=100, height=100 +After: x=10, y=10, width=100, height=100 diff --git a/tests/examples/text/gears/object/properties.lua b/tests/examples/text/gears/object/properties.lua index ffb05ddca..09b82bf32 100644 --- a/tests/examples/text/gears/object/properties.lua +++ b/tests/examples/text/gears/object/properties.lua @@ -23,7 +23,7 @@ function class:set_foo(value) end function class:method(a, b, c) - print("In a mathod", a, b, c) + print("In a method", a, b, c) end local o = gears.object { diff --git a/tests/examples/text/gears/object/properties.output.txt b/tests/examples/text/gears/object/properties.output.txt new file mode 100644 index 000000000..16f4a60dd --- /dev/null +++ b/tests/examples/text/gears/object/properties.output.txt @@ -0,0 +1,9 @@ +In get foo bar +bar +In set foo 42 +In get foo 42 +42 +In a method 1 2 3 +nil +In the connection handler! a cow +a cow diff --git a/tests/examples/text/gears/object/signal.output.txt b/tests/examples/text/gears/object/signal.output.txt new file mode 100644 index 000000000..c73dae3ac --- /dev/null +++ b/tests/examples/text/gears/object/signal.output.txt @@ -0,0 +1,2 @@ +In slot [obj] nil nil nil +In slot [obj] foo bar 42 diff --git a/tests/examples/text/gears/sort/topological.output.txt b/tests/examples/text/gears/sort/topological.output.txt new file mode 100644 index 000000000..8c2c033b2 --- /dev/null +++ b/tests/examples/text/gears/sort/topological.output.txt @@ -0,0 +1,6 @@ +The position #1 is: a +The position #2 is: b +The position #3 is: c +The position #4 is: d +The position #5 is: e +The position #6 is: f diff --git a/tests/examples/wibox/layout/grid/add.output.txt b/tests/examples/wibox/layout/grid/add.output.txt new file mode 100644 index 000000000..aa38c48a0 --- /dev/null +++ b/tests/examples/wibox/layout/grid/add.output.txt @@ -0,0 +1 @@ +l:add_widget_at(new, 1, 4, 1, 1) diff --git a/tests/examples/wibox/layout/grid/extend_column.output.txt b/tests/examples/wibox/layout/grid/extend_column.output.txt new file mode 100644 index 000000000..ac9b51d8a --- /dev/null +++ b/tests/examples/wibox/layout/grid/extend_column.output.txt @@ -0,0 +1 @@ +l:extend_column(2) diff --git a/tests/examples/wibox/layout/grid/insert_column.output.txt b/tests/examples/wibox/layout/grid/insert_column.output.txt new file mode 100644 index 000000000..3c10daf8d --- /dev/null +++ b/tests/examples/wibox/layout/grid/insert_column.output.txt @@ -0,0 +1 @@ +l:insert_column(2) diff --git a/tests/examples/wibox/layout/grid/orientation.lua b/tests/examples/wibox/layout/grid/orientation.lua index 8a36679e1..d0274925b 100644 --- a/tests/examples/wibox/layout/grid/orientation.lua +++ b/tests/examples/wibox/layout/grid/orientation.lua @@ -10,8 +10,7 @@ print([[l = wibox.layout { layout = wibox.layout.grid } l:set_orientation("vertical") -- change to "horizontal" -l:add(...) -]]) --DOC_HIDE +l:add(...)]]) --DOC_HIDE return --DOC_HIDE wibox.widget { diff --git a/tests/examples/wibox/layout/grid/orientation.output.txt b/tests/examples/wibox/layout/grid/orientation.output.txt new file mode 100644 index 000000000..cb19909a7 --- /dev/null +++ b/tests/examples/wibox/layout/grid/orientation.output.txt @@ -0,0 +1,8 @@ +l = wibox.layout { + forced_num_cols = 2, + forced_num_rows = 2, + homogeneous = true, + layout = wibox.layout.grid +} +l:set_orientation("vertical") -- change to "horizontal" +l:add(...) diff --git a/tests/examples/wibox/layout/grid/remove.output.txt b/tests/examples/wibox/layout/grid/remove.output.txt new file mode 100644 index 000000000..f4d239e87 --- /dev/null +++ b/tests/examples/wibox/layout/grid/remove.output.txt @@ -0,0 +1 @@ +l:remove_widgets_at(1,1) diff --git a/tests/examples/wibox/layout/grid/remove_column.output.txt b/tests/examples/wibox/layout/grid/remove_column.output.txt new file mode 100644 index 000000000..b8e5d85e5 --- /dev/null +++ b/tests/examples/wibox/layout/grid/remove_column.output.txt @@ -0,0 +1 @@ +l:remove_column(2) From 477af1b14540e61080feb06211b01ba12998a7ab Mon Sep 17 00:00:00 2001 From: Uli Schlachter Date: Wed, 25 Jul 2018 16:48:29 +0200 Subject: [PATCH 2/2] test runner: Move running to a shell script Instead of running the example tests directly from CMake and checking the results via CMake, the tests are now run through a shell script. So far, there are too many variables involved for me to easily figure out how to run this shell script in the building phase instead of the configuring phase, but at least this commit moves the "actual running" out of CMake, bringing us a step closer to that goal. Signed-off-by: Uli Schlachter --- tests/examples/CMakeLists.txt | 23 ++++++++-------- tests/examples/runner.sh | 51 +++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 12 deletions(-) create mode 100755 tests/examples/runner.sh diff --git a/tests/examples/CMakeLists.txt b/tests/examples/CMakeLists.txt index 4cb869f6e..2f45b84bc 100644 --- a/tests/examples/CMakeLists.txt +++ b/tests/examples/CMakeLists.txt @@ -188,6 +188,14 @@ function(run_test test_path namespace escaped_content) set(DOC_BLOCK_PREFIX "@usage") endif() + # Does the text generate text output? + if(tmp_content MATCHES "--DOC_GEN_OUTPUT") + # The expected output is next to the .lua file in a .output.txt file + string(REPLACE ".lua" ".output.txt" expected_output_path ${test_path}) + else() + set(expected_output_path "/dev/null") + endif() + # Get the file name without the extension. get_filename_component(${test_path} TEST_FILE_NAME NAME) set(IMAGE_PATH "${IMAGE_DIR}/AUTOGEN${namespace}_${TEST_FILE_NAME}") @@ -197,12 +205,12 @@ function(run_test test_path namespace escaped_content) message(STATUS "Running ${rel_test_path}…") execute_process( - COMMAND ${LUA_COV_RUNNER} ${template} ${test_path} ${IMAGE_PATH} + COMMAND "${TOP_SOURCE_DIR}/tests/examples/runner.sh" "${expected_output_path}" ${LUA_COV_RUNNER} ${template} ${test_path} ${IMAGE_PATH} RESULT_VARIABLE TEST_RESULT OUTPUT_VARIABLE TEST_OUTPUT ERROR_VARIABLE TEST_ERROR ) - if (TEST_RESULT OR NOT TEST_ERROR STREQUAL "") + if (TEST_RESULT OR NOT TEST_OUTPUT STREQUAL "" OR NOT TEST_ERROR STREQUAL "") message("Result: ${TEST_RESULT}") if (NOT TEST_OUTPUT STREQUAL "") message("Output: ${TEST_OUTPUT}") @@ -236,12 +244,7 @@ function(run_test test_path namespace escaped_content) # If there is an output, assume it is relevant and add it to the # documentation under the image. if(tmp_content MATCHES "--DOC_GEN_OUTPUT") - # The expected output is next to the .lua file in a .output.txt file - string(REPLACE ".lua" ".output.txt" expected_output_path ${test_path}) file(READ "${expected_output_path}" expected_output) - if(NOT "${TEST_OUTPUT}" STREQUAL "${expected_output}") - message(SEND_ERROR "Wrong output from ${test_path}:\nExpected:\n${expected_output}\nGot:\n${TEST_OUTPUT}") - endif() set(TEST_DOC_CONTENT "${TEST_DOC_CONTENT}\n${DOC_LINE_PREFIX}\n${DOC_LINE_PREFIX}**Usage example output**:\n${DOC_LINE_PREFIX}" @@ -249,14 +252,10 @@ function(run_test test_path namespace escaped_content) # Markdown requires an empty line before and after, and 4 spaces. escape_string( - "\n${TEST_OUTPUT}" + "\n${expected_output}" "${TEST_DOC_CONTENT}" TEST_DOC_CONTENT " " ) set(TEST_DOC_CONTENT "${TEST_DOC_CONTENT}\n${DOC_LINE_PREFIX}") - else() - if(NOT ${TEST_OUTPUT} STREQUAL "") - message(FATAL_ERROR "Unexpected output from ${test_path}: ${TEST_OUTPUT}") - endif() endif() # If there is some @* content, append it. diff --git a/tests/examples/runner.sh b/tests/examples/runner.sh new file mode 100755 index 000000000..d3eb820c9 --- /dev/null +++ b/tests/examples/runner.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# Todo: Give this shell script a sane calling convention. Right now it has a +# file containing the expected output as first argument and just executes +# everything else. + +set -ue + +# Cleanup on errors / aborting +cleanup() { + rm -rf "$file_stderr" "$file_stdout" || true +} +trap "cleanup" 0 2 3 15 + +file_stdout=$(mktemp) +file_stderr=$(mktemp) + +expected_output=$1 +shift + +# Run the command that we were given +exit_code=0 +"$@" > ${file_stdout} 2> ${file_stderr} || exit_code=$? + +# If exit code is not zero or anything was produced on stderr... +if [ $exit_code -ne 0 -o -s "${file_stderr}" ] +then + echo "Result: ${exit_code}" + if [ -s "${file_stdout}" ] + then + echo "Output:" + cat "${file_stdout}" + fi + if [ -s "${file_stderr}" ] + then + echo "Error:" + cat "${file_stderr}" + fi + exit 1 +fi + +# Check if we got the output we wanted +if ! cmp --silent "${file_stdout}" "${expected_output}" +then + echo "Expected text from ${expected_output}, but got:" + diff -u "${expected_output}" "${file_stdout}" || true + exit 1 +fi + +exit 0 + +# vim: filetype=sh:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:textwidth=80