testrunner.h
#ifndef WREPORT_TESTSRUNNER_H
#define WREPORT_TESTSRUNNER_H
#include <string>
#include <vector>
#include <functional>
#include <memory>
namespace wreport {
namespace term {
struct Terminal;
}
namespace tests {
struct TestFailed;
struct TestStack;
struct TestCase;
struct TestMethod;
struct TestMethodResult
{
std::string test_case;
std::string test_method;
std::string error_message;
std::shared_ptr<TestStack> error_stack;
std::string exception_typeid;
bool skipped = false;
std::string skipped_reason;
unsigned long long elapsed_ns = 0;
TestMethodResult(const std::string& test_case, const std::string& test_method)
: test_case(test_case), test_method(test_method) {}
void set_failed(TestFailed& e);
void set_exception(std::exception& e)
{
error_message = e.what();
if (error_message.empty())
error_message = "test threw an exception with an empty error message";
exception_typeid = typeid(e).name();
}
void set_unknown_exception()
{
error_message = "unknown exception caught";
}
void set_setup_exception(std::exception& e)
{
error_message = "[setup failed: ";
error_message += e.what();
error_message += "]";
}
void set_teardown_exception(std::exception& e)
{
error_message = "[teardown failed: ";
error_message += e.what();
error_message += "]";
}
bool is_success() const
{
return error_message.empty();
}
void print_failure_details(FILE* out) const;
};
struct TestCaseResult
{
std::string test_case;
std::vector<TestMethodResult> methods;
std::string fail_setup;
std::string fail_teardown;
bool skipped = false;
TestCaseResult(const std::string& test_case) : test_case(test_case) {}
void set_setup_failed()
{
fail_setup = "test case setup method threw an unknown exception";
}
void set_setup_failed(std::exception& e)
{
fail_setup = "test case setup method threw an exception: ";
fail_setup += e.what();
}
void set_teardown_failed()
{
fail_teardown = "test case teardown method threw an unknown exception";
}
void set_teardown_failed(std::exception& e)
{
fail_teardown = "test case teardown method threw an exception: ";
fail_teardown += e.what();
}
void add_test_method(TestMethodResult&& e)
{
methods.emplace_back(std::move(e));
}
bool is_success() const
{
if (!fail_setup.empty() || !fail_teardown.empty()) return false;
for (const auto& m: methods)
if (!m.is_success())
return false;
return true;
}
unsigned long long elapsed_ns() const;
};
struct TestController
{
virtual ~TestController() {}
virtual bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) { return true; }
virtual void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) {}
virtual bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) { return true; }
virtual void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) {}
};
struct FilteringTestController : public TestController
{
std::string allowlist;
std::string blocklist;
bool test_method_should_run(const std::string& fullname) const;
};
struct SimpleTestController : public FilteringTestController
{
wreport::term::Terminal& output;
SimpleTestController(wreport::term::Terminal& output);
bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) override;
void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) override;
bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
};
struct VerboseTestController : public FilteringTestController
{
wreport::term::Terminal& output;
VerboseTestController(wreport::term::Terminal& output);
bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) override;
void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) override;
bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
};
struct TestRegistry
{
std::vector<TestCase*> entries;
void register_test_case(TestCase& test_case);
void iterate_test_methods(std::function<void(const TestCase&, const TestMethod&)>);
std::vector<TestCaseResult> run_tests(TestController& controller);
static TestRegistry& get();
};
struct TestResultStats
{
const std::vector<TestCaseResult>& results;
unsigned methods_ok = 0;
unsigned methods_failed = 0;
unsigned methods_skipped = 0;
unsigned test_cases_ok = 0;
unsigned test_cases_failed = 0;
bool success = false;
TestResultStats(const std::vector<TestCaseResult>& results);
void print_results(wreport::term::Terminal& out);
void print_stats(wreport::term::Terminal& out);
void print_summary(wreport::term::Terminal& out);
};
}
}
#endif
wreport
String functions.
Definition benchmark.h:13
wreport::term::Terminal
Definition term.h:18
wreport::tests::FilteringTestController
Test controller that filters tests via a blocklist/allowlist system containing glob patterns on testc...
Definition testrunner.h:192
wreport::tests::FilteringTestController::blocklist
std::string blocklist
Any method matching this glob expression will not be run.
Definition testrunner.h:197
wreport::tests::FilteringTestController::allowlist
std::string allowlist
Any method not matching this glob expression will not be run.
Definition testrunner.h:194
wreport::tests::SimpleTestController::test_method_end
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
wreport::tests::SimpleTestController::test_method_begin
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.
wreport::tests::SimpleTestController::test_case_end
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
wreport::tests::SimpleTestController::test_case_begin
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
wreport::tests::TestCaseResult
Result of running a whole test case.
Definition testrunner.h:97
wreport::tests::TestCaseResult::fail_setup
std::string fail_setup
Set to a non-empty string if the setup method of the test case failed.
Definition testrunner.h:103
wreport::tests::TestCaseResult::fail_teardown
std::string fail_teardown
Set to a non-empty string if the teardown method of the test case failed.
Definition testrunner.h:106
wreport::tests::TestCaseResult::methods
std::vector< TestMethodResult > methods
Outcome of all the methods that have been run.
Definition testrunner.h:101
wreport::tests::TestCaseResult::skipped
bool skipped
Set to true if this test case has been skipped.
Definition testrunner.h:108
wreport::tests::TestCaseResult::test_case
std::string test_case
Name of the test case.
Definition testrunner.h:99
wreport::tests::TestCase
Test case collecting several test methods, and self-registering with the singleton instance of TestRe...
Definition utils/tests.h:520
wreport::tests::TestController
Abstract interface for the objects that supervise test execution.
Definition testrunner.h:159
wreport::tests::TestController::test_case_end
virtual void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result)
Called after running a test case.
Definition testrunner.h:172
wreport::tests::TestController::test_case_begin
virtual bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result)
Called before running a test case.
Definition testrunner.h:167
wreport::tests::TestController::test_method_begin
virtual bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result)
Called before running a test method.
Definition testrunner.h:179
wreport::tests::TestController::test_method_end
virtual void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result)
Called after running a test method.
Definition testrunner.h:184
wreport::tests::TestFailed
Exception thrown when a test assertion fails, normally by Location::fail_test.
Definition utils/tests.h:104
wreport::tests::TestMethodResult
Result of running a test method.
Definition testrunner.h:27
wreport::tests::TestMethodResult::skipped_reason
std::string skipped_reason
If the test has been skipped, this is an optional reason.
Definition testrunner.h:47
wreport::tests::TestMethodResult::error_stack
std::shared_ptr< TestStack > error_stack
Stack frame of where the error happened.
Definition testrunner.h:38
wreport::tests::TestMethodResult::test_case
std::string test_case
Name of the test case.
Definition testrunner.h:29
wreport::tests::TestMethodResult::skipped
bool skipped
True if the test has been skipped.
Definition testrunner.h:44
wreport::tests::TestMethodResult::error_message
std::string error_message
If non-empty, the test failed with this error.
Definition testrunner.h:35
wreport::tests::TestMethodResult::elapsed_ns
unsigned long long elapsed_ns
Time in nanoseconds it took the test to run.
Definition testrunner.h:50
wreport::tests::TestMethodResult::exception_typeid
std::string exception_typeid
If non-empty, the test threw an exception and this is its type ID.
Definition testrunner.h:41
wreport::tests::TestMethodResult::test_method
std::string test_method
Name of the test method.
Definition testrunner.h:32
wreport::tests::TestMethod
Test method information.
Definition utils/tests.h:493
wreport::tests::TestRegistry
Test registry.
Definition testrunner.h:248
wreport::tests::TestRegistry::iterate_test_methods
void iterate_test_methods(std::function< void(const TestCase &, const TestMethod &)>)
Iterate on all test methods known by this registry.
wreport::tests::TestRegistry::run_tests
std::vector< TestCaseResult > run_tests(TestController &controller)
Run all the registered tests using the given controller.
wreport::tests::TestRegistry::entries
std::vector< TestCase * > entries
All known test cases.
Definition testrunner.h:250
wreport::tests::TestRegistry::register_test_case
void register_test_case(TestCase &test_case)
Register a new test case.
wreport::tests::TestRegistry::get
static TestRegistry & get()
Get the singleton instance of TestRegistry.
wreport::tests::TestStack
Definition utils/tests.h:89
wreport::tests::VerboseTestController::test_method_end
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
wreport::tests::VerboseTestController::test_case_begin
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
wreport::tests::VerboseTestController::test_case_end
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
wreport::tests::VerboseTestController::test_method_begin
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.