| 215 | } |
| 216 | |
| 217 | static void evaluateTestCase(const std::set<std::string>& skipList, |
| 218 | std::string harnessDirectory, |
| 219 | std::string testFile, |
| 220 | const TestCaseData& metadata, |
| 221 | bool isStrict, |
| 222 | TestReporter& reporter) { |
| 223 | const auto lastDelimiterPos = testFile.find_last_of('/'); |
| 224 | const auto testSuite = testFile.substr(0, lastDelimiterPos); |
| 225 | const auto testName = testFile.substr(lastDelimiterPos + 1, testFile.size()); |
| 226 | |
| 227 | if (skipList.contains(testFile)) { |
| 228 | reporter.submitDisabled(testSuite, testName, "Skipping tests case due to known execution crash", isStrict); |
| 229 | return; |
| 230 | } |
| 231 | |
| 232 | if (!tsn_contains_module(testFile.c_str())) { |
| 233 | reporter.submitDisabled(testSuite, testName, "Skipping tests case due to compilation failure", isStrict); |
| 234 | return; |
| 235 | } |
| 236 | |
| 237 | std::string error_type = ""; |
| 238 | std::string error_description = ""; |
| 239 | |
| 240 | auto result = |
| 241 | evaluateTestCaseInternal(harnessDirectory, testFile, metadata, isStrict, error_type, error_description); |
| 242 | |
| 243 | switch (result) { |
| 244 | case ExecutionResult::RunWithPass: { |
| 245 | reporter.submitPass(testSuite, testName, isStrict); |
| 246 | } break; |
| 247 | case ExecutionResult::RunWithError: { |
| 248 | if (metadata.isNegative) { |
| 249 | reporter.submitPass(testSuite, testName, isStrict); |
| 250 | } else { |
| 251 | reporter.submitFailure( |
| 252 | testSuite, |
| 253 | testName, |
| 254 | fmt::format("Expected: \"{}\" returned: \"{}\" - ", metadata.negativeType, error_type), |
| 255 | isStrict); |
| 256 | } |
| 257 | } break; |
| 258 | case ExecutionResult::Failed: { |
| 259 | reporter.submitFailure(testSuite, testName, fmt::format("Error: {}", error_type), isStrict); |
| 260 | } break; |
| 261 | } |
| 262 | } |
| 263 | |
| 264 | bool runTestCases(const fs::path& harnessDirectory, |
| 265 | const fs::path& testCaseFile, |
no test coverage detected