Elaboradar  0.1
testrunner.h
1 #ifndef RADARELAB_UTILS_TESTSRUNNER_H
2 #define RADARELAB_UTILS_TESTSRUNNER_H
3 
4 #include <string>
5 #include <vector>
6 #include <functional>
7 #include <memory>
8 
9 namespace radarelab {
10 namespace utils {
11 
12 namespace term {
13 struct Terminal;
14 }
15 
16 namespace tests {
17 
18 struct TestFailed;
19 struct TestStack;
20 struct TestCase;
21 struct TestMethod;
22 
23 
28 {
30  std::string test_case;
31 
33  std::string test_method;
34 
36  std::string error_message;
37 
39  std::shared_ptr<TestStack> error_stack;
40 
42  std::string exception_typeid;
43 
45  bool skipped = false;
46 
48  std::string skipped_reason;
49 
51  unsigned long long elapsed_ns = 0;
52 
53 
54  TestMethodResult(const std::string& test_case, const std::string& test_method)
56 
57  void set_failed(TestFailed& e);
58 
59  void set_exception(std::exception& e)
60  {
61  error_message = e.what();
62  if (error_message.empty())
63  error_message = "test threw an exception with an empty error message";
64  exception_typeid = typeid(e).name();
65  }
66 
67  void set_unknown_exception()
68  {
69  error_message = "unknown exception caught";
70  }
71 
72  void set_setup_exception(std::exception& e)
73  {
74  error_message = "[setup failed: ";
75  error_message += e.what();
76  error_message += "]";
77  }
78 
79  void set_teardown_exception(std::exception& e)
80  {
81  error_message = "[teardown failed: ";
82  error_message += e.what();
83  error_message += "]";
84  }
85 
86  bool is_success() const
87  {
88  return error_message.empty();
89  }
90 
91  void print_failure_details(FILE* out) const;
92 };
93 
98 {
100  std::string test_case;
102  std::vector<TestMethodResult> methods;
104  std::string fail_setup;
107  std::string fail_teardown;
109  bool skipped = false;
110 
111  TestCaseResult(const std::string& test_case) : test_case(test_case) {}
112 
113  void set_setup_failed()
114  {
115  fail_setup = "test case setup method threw an unknown exception";
116  }
117 
118  void set_setup_failed(std::exception& e)
119  {
120  fail_setup = "test case setup method threw an exception: ";
121  fail_setup += e.what();
122  }
123 
124  void set_teardown_failed()
125  {
126  fail_teardown = "test case teardown method threw an unknown exception";
127  }
128 
129  void set_teardown_failed(std::exception& e)
130  {
131  fail_teardown = "test case teardown method threw an exception: ";
132  fail_teardown += e.what();
133  }
134 
135  void add_test_method(TestMethodResult&& e)
136  {
137  methods.emplace_back(std::move(e));
138  }
139 
140  bool is_success() const
141  {
142  if (!fail_setup.empty() || !fail_teardown.empty()) return false;
143  for (const auto& m: methods)
144  if (!m.is_success())
145  return false;
146  return true;
147  }
148 
149  unsigned long long elapsed_ns() const;
150 };
151 
152 
160 {
161  virtual ~TestController() {}
162 
168  virtual bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) { return true; }
169 
173  virtual void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) {}
174 
180  virtual bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) { return true; }
181 
185  virtual void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) {}
186 };
187 
193 {
195  std::string allowlist;
196 
198  std::string blocklist;
199 
200  bool test_method_should_run(const std::string& fullname) const;
201 };
202 
203 
211 {
212  radarelab::utils::term::Terminal& output;
213 
214  SimpleTestController(radarelab::utils::term::Terminal& output);
215 
216  bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) override;
217  void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) override;
218  bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
219  void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
220 };
221 
222 
230 {
231  radarelab::utils::term::Terminal& output;
232 
233  VerboseTestController(radarelab::utils::term::Terminal& output);
234 
235  bool test_case_begin(const TestCase& test_case, const TestCaseResult& test_case_result) override;
236  void test_case_end(const TestCase& test_case, const TestCaseResult& test_case_result) override;
237  bool test_method_begin(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
238  void test_method_end(const TestMethod& test_method, const TestMethodResult& test_method_result) override;
239 };
240 
241 
249 {
251  std::vector<TestCase*> entries;
252 
259  void register_test_case(TestCase& test_case);
260 
267  void iterate_test_methods(std::function<void(const TestCase&, const TestMethod&)>);
268 
272  std::vector<TestCaseResult> run_tests(TestController& controller);
273 
275  static TestRegistry& get();
276 };
277 
278 
279 struct TestResultStats
280 {
281  const std::vector<TestCaseResult>& results;
282  unsigned methods_ok = 0;
283  unsigned methods_failed = 0;
284  unsigned methods_skipped = 0;
285  unsigned test_cases_ok = 0;
286  unsigned test_cases_failed = 0;
287  bool success = false;
288 
289  TestResultStats(const std::vector<TestCaseResult>& results);
290 
291  void print_results(radarelab::utils::term::Terminal& out);
292  void print_stats(radarelab::utils::term::Terminal& out);
293  void print_summary(radarelab::utils::term::Terminal& out);
294 };
295 
296 }
297 }
298 }
299 #endif
String functions.
Definition: cart.cpp:4
std::string allowlist
Any method not matching this glob expression will not be run.
Definition: testrunner.h:195
std::string blocklist
Any method matching this glob expression will not be run.
Definition: testrunner.h:198
Test controller that filters tests via a blocklist/allowlist system containing glob patterns on testc...
Definition: testrunner.h:193
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.
Simple default implementation of TestController.
Definition: testrunner.h:211
std::string fail_teardown
Set to a non-empty string if the teardown method of the test case failed.
Definition: testrunner.h:107
std::vector< TestMethodResult > methods
Outcome of all the methods that have been run.
Definition: testrunner.h:102
std::string test_case
Name of the test case.
Definition: testrunner.h:100
std::string fail_setup
Set to a non-empty string if the setup method of the test case failed.
Definition: testrunner.h:104
bool skipped
Set to true if this test case has been skipped.
Definition: testrunner.h:109
Result of running a whole test case.
Definition: testrunner.h:98
Test case collecting several test methods, and self-registering with the singleton instance of TestRe...
Definition: tests.h:523
virtual bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result)
Called before running a test case.
Definition: testrunner.h:168
virtual void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result)
Called after running a test case.
Definition: testrunner.h:173
virtual bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result)
Called before running a test method.
Definition: testrunner.h:180
virtual void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result)
Called after running a test method.
Definition: testrunner.h:185
Abstract interface for the objects that supervise test execution.
Definition: testrunner.h:160
Exception thrown when a test assertion fails, normally by Location::fail_test.
Definition: tests.h:107
std::string test_method
Name of the test method.
Definition: testrunner.h:33
std::string skipped_reason
If the test has been skipped, this is an optional reason.
Definition: testrunner.h:48
std::shared_ptr< TestStack > error_stack
Stack frame of where the error happened.
Definition: testrunner.h:39
bool skipped
True if the test has been skipped.
Definition: testrunner.h:45
std::string test_case
Name of the test case.
Definition: testrunner.h:30
unsigned long long elapsed_ns
Time in nanoseconds it took the test to run.
Definition: testrunner.h:51
std::string exception_typeid
If non-empty, the test threw an exception and this is its type ID.
Definition: testrunner.h:42
std::string error_message
If non-empty, the test failed with this error.
Definition: testrunner.h:36
Result of running a test method.
Definition: testrunner.h:28
Test method information.
Definition: tests.h:496
void iterate_test_methods(std::function< void(const TestCase &, const TestMethod &)>)
Iterate on all test methods known by this registry.
static TestRegistry & get()
Get the singleton instance of TestRegistry.
std::vector< TestCaseResult > run_tests(TestController &controller)
Run all the registered tests using the given controller.
void register_test_case(TestCase &test_case)
Register a new test case.
std::vector< TestCase * > entries
All known test cases.
Definition: testrunner.h:251
void test_case_end(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called after running a test case.
bool test_case_begin(const TestCase &test_case, const TestCaseResult &test_case_result) override
Called before running a test case.
bool test_method_begin(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called before running a test method.
void test_method_end(const TestMethod &test_method, const TestMethodResult &test_method_result) override
Called after running a test method.
Verbose implementation of TestController.
Definition: testrunner.h:230